Merge -dagss and -devel

Cython/Compiler/Builtin.py

@@ -82,7 +82,7 @@ builtin_function_table = [
 builtin_types_table = [
 
     ("type",    "PyType_Type",     []),
-#    ("str",     "PyString_Type",   []),
+#    ("str",     "PyBytes_Type",   []),
     ("unicode", "PyUnicode_Type",  []),
     ("file",    "PyFile_Type",     []),
 #    ("slice",   "PySlice_Type",    []),

Cython/Compiler/CmdLine.py

@@ -17,6 +17,10 @@ Options:
   -I, --include-dir <directory>  Search for include files in named directory
                                  (multiply include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
+  -r, --recursive                Recursively find and compile dependencies
+  -t, --timestamps               Only compile newer source files (implied with -r)
+  -f, --force                    Compile all source files (overrides implied -t)
+  -q, --quiet                    Don't print module names in recursive mode
   -p, --embed-positions          If specified, the positions in Cython files of each
                                  function definition is embedded in its docstring.
   -z, --pre-import <module>      If specified, assume undeclared names in this 
@@ -111,6 +115,12 @@ def parse_command_line(args):
                 options.working_path = pop_arg()
             elif option in ("-o", "--output-file"):
                 options.output_file = pop_arg()
+            elif option in ("-r", "--recursive"):
+                options.recursive = 1
+            elif option in ("-t", "--timestamps"):
+                options.timestamps = 1
+            elif option in ("-f", "--force"):
+                options.timestamps = 0
             elif option in ("-p", "--embed-positions"):
                 Options.embed_pos_in_docstring = 1
             elif option in ("-z", "--pre-import"):

Cython/Compiler/ModuleNode.py

@@ -54,6 +54,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if self.has_imported_c_functions():
             self.module_temp_cname = env.allocate_temp_pyobject()
             env.release_temp(self.module_temp_cname)
+        self.generate_dep_file(env, result)
         self.generate_c_code(env, options, result)
         self.generate_h_code(env, options, result)
         self.generate_api_code(env, result)
@@ -65,6 +66,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     return 1
         return 0
     
+    def generate_dep_file(self, env, result):
+        modules = self.referenced_modules
+        if len(modules) > 1 or env.included_files:
+            dep_file = replace_suffix(result.c_file, ".dep")
+            f = open(dep_file, "w")
+            try:
+                for module in modules:
+                    if module is not env:
+                        f.write("cimport %s\n" % module.qualified_name)
+                    for path in module.included_files:
+                        f.write("include %s\n" % path)
+            finally:
+                f.close()
+
     def generate_h_code(self, env, options, result):
         def h_entries(entries, pxd = 0):
             return [entry for entry in entries
@@ -348,12 +363,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
     def generate_declarations_for_modules(self, env, modules, code):
         code.putln("")
-        code.putln("/* Declarations */")
+        code.putln("/* Type declarations */")
         vtab_list, vtabslot_list = self.sort_type_hierarchy(modules, env)
         self.generate_type_definitions(
             env, modules, vtab_list, vtabslot_list, code)
         for module in modules:
             defined_here = module is env
+            code.putln("/* Module declarations from %s */" %
+                       module.qualified_name.encode("ASCII", "ignore"))
             self.generate_global_declarations(module, code, defined_here)
             self.generate_cfunction_predeclarations(module, code, defined_here)
 
@@ -440,6 +457,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask")
         code.putln("  #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask")
         code.putln("  #define PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
+        code.putln("#else")
+        code.putln("  #define PyBytes_Type                 PyString_Type")
         code.putln("#endif")
 
         code.putln("#if PY_MAJOR_VERSION >= 3")
@@ -686,8 +705,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     entry.type.typeptr_cname)
         code.put_var_declarations(env.var_entries, static = 1, 
             dll_linkage = "DL_EXPORT", definition = definition)
-        code.put_var_declarations(env.default_entries, static = 1,
-                                  definition = definition)
+        if definition:
+            code.put_var_declarations(env.default_entries, static = 1,
+                                      definition = definition)
     
     def generate_cfunction_predeclarations(self, env, code, definition):
         for entry in env.cfunc_entries:
@@ -2124,8 +2144,13 @@ __Pyx_import_all_from(PyObject *locals, PyObject *v)
 			break;
 		}
 		if (skip_leading_underscores &&
+#if PY_MAJOR_VERSION < 3
 		    PyString_Check(name) &&
 		    PyString_AS_STRING(name)[0] == '_')
+#else
+		    PyUnicode_Check(name) &&
+		    PyUnicode_AS_UNICODE(name)[0] == '_')
+#endif
 		{
 			Py_DECREF(name);
 			continue;
@@ -2147,10 +2172,11 @@ __Pyx_import_all_from(PyObject *locals, PyObject *v)
 }
 
 
-static int %s(PyObject* m) {
+static int %(IMPORT_STAR)s(PyObject* m) {
 
     int i;
     int ret = -1;
+    char* s;
     PyObject *locals = 0;
     PyObject *list = 0;
     PyObject *name;
@@ -2163,7 +2189,13 @@ static int %s(PyObject* m) {
     for(i=0; i<PyList_GET_SIZE(list); i++) {
         name = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 0);
         item = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 1);
-        if (%s(item, name, PyString_AsString(name)) < 0) goto bad;
+#if PY_MAJOR_VERSION < 3
+        s = PyString_AsString(name);
+#else
+        s = PyUnicode_AsString(name);
+#endif
+        if (!s) goto bad;
+        if (%(IMPORT_STAR_SET)s(item, name, s) < 0) goto bad;
     }
     ret = 0;
     
@@ -2172,4 +2204,5 @@ bad:
     Py_XDECREF(list);
     return ret;
 }
-""" % ( Naming.import_star, Naming.import_star_set )
+""" % {'IMPORT_STAR'     : Naming.import_star,
+       'IMPORT_STAR_SET' : Naming.import_star_set }

Cython/Compiler/PyrexTypes.py

@@ -695,9 +695,10 @@ class CFuncType(CType):
             return 0
         if not self.same_calling_convention_as(other_type):
             return 0
+        if self.nogil and not other_type.nogil:
+            return 0
         return 1
-        
-    
+
     def compatible_signature_with(self, other_type, as_cmethod = 0):
         return self.compatible_signature_with_resolved_type(other_type.resolve(), as_cmethod)
     
@@ -789,22 +790,24 @@ class CFuncType(CType):
         arg_decl_code = ", ".join(arg_decl_list)
         if not arg_decl_code and not pyrex:
             arg_decl_code = "void"
-        exc_clause = ""
+        trailer = ""
         if (pyrex or for_display) and not self.return_type.is_pyobject:
             if self.exception_value and self.exception_check:
-                exc_clause = " except? %s" % self.exception_value
+                trailer = " except? %s" % self.exception_value
             elif self.exception_value:
-                exc_clause = " except %s" % self.exception_value
+                trailer = " except %s" % self.exception_value
             elif self.exception_check == '+':
-                exc_clause = " except +"
+                trailer = " except +"
             else:
-                " except *"
+                " except *" # ignored
+            if self.nogil:
+                trailer += " nogil"
         cc = self.calling_convention_prefix()
         if (not entity_code and cc) or entity_code.startswith("*"):
             entity_code = "(%s%s)" % (cc, entity_code)
             cc = ""
         return self.return_type.declaration_code(
-            "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, exc_clause),
+            "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, trailer),
             for_display, dll_linkage, pyrex)
 	
     def function_header_code(self, func_name, arg_code):

Cython/Compiler/Scanning.py

@@ -284,11 +284,10 @@ class PyrexScanner(Scanner):
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    
     resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
-            type_names = None, context = None, source_encoding=None):
+                 type_names = None, context = None, source_encoding=None):
         Scanner.__init__(self, get_lexicon(), file, filename)
         if parent_scanner:
             self.context = parent_scanner.context

Cython/Compiler/Symtab.py

@@ -150,12 +150,14 @@ class Scope:
     # pystring_entries  [Entry]            String const entries newly used as
     #                                        Python strings in this scope
     # control_flow     ControlFlow  Used for keeping track of environment state
+    # nogil             boolean            In a nogil section
 
     is_py_class_scope = 0
     is_c_class_scope = 0
     is_module_scope = 0
     scope_prefix = ""
     in_cinclude = 0
+    nogil = 0
     
     def __init__(self, name, outer_scope, parent_scope):
         # The outer_scope is the next scope in the lookup chain.
@@ -554,10 +556,6 @@ class Scope:
         return [entry for entry in self.temp_entries
             if entry not in self.free_temp_entries]
     
-    #def recycle_pending_temps(self):
-    #	# Obsolete
-    #	pass
-
     def use_utility_code(self, new_code):
         self.global_scope().use_utility_code(new_code)
     
@@ -652,7 +650,7 @@ class BuiltinScope(Scope):
         "long":   ["((PyObject*)&PyLong_Type)", py_object_type],
         "float":  ["((PyObject*)&PyFloat_Type)", py_object_type],
         
-        "str":    ["((PyObject*)&PyString_Type)", py_object_type],
+        "str":    ["((PyObject*)&PyBytes_Type)", py_object_type],
         "unicode":["((PyObject*)&PyUnicode_Type)", py_object_type],
         "tuple":  ["((PyObject*)&PyTuple_Type)", py_object_type],
         "list":   ["((PyObject*)&PyList_Type)", py_object_type],
@@ -687,6 +685,7 @@ class ModuleScope(Scope):
     # parent_module        Scope              Parent in the import namespace
     # module_entries       {string : Entry}   For cimport statements
     # type_names           {string : 1}       Set of type names (used during parsing)
+    # included_files       [string]           Cython sources included with 'include'
     # pxd_file_loaded      boolean            Corresponding .pxd file has been processed
     # cimported_modules    [ModuleScope]      Modules imported with cimport
     # new_interned_string_entries [Entry]     New interned strings waiting to be declared
@@ -723,6 +722,7 @@ class ModuleScope(Scope):
         self.interned_objs = []
         self.all_pystring_entries = []
         self.types_imported = {}
+        self.included_files = []
         self.pynum_entries = []
         self.has_extern_class = 0
         self.cached_builtins = []
@@ -1326,6 +1326,7 @@ class CClassScope(ClassScope):
 #                    entry.type = type
                 else:
                     error(pos, "Signature not compatible with previous declaration")
+                    error(entry.pos, "Previous declaration is here")
         else:
             if self.defined:
                 error(pos,
@@ -1365,8 +1366,8 @@ class CClassScope(ClassScope):
                 entry.is_variable = 1
                 self.inherited_var_entries.append(entry)
         for base_entry in base_scope.cfunc_entries:
-            entry = self.add_cfunction(base_entry.name, base_entry.type, None,
-                adapt(base_entry.cname), base_entry.visibility)
+            entry = self.add_cfunction(base_entry.name, base_entry.type,
+                    base_entry.pos, adapt(base_entry.cname), base_entry.visibility)
             entry.is_inherited = 1
             
     def allocate_temp(self, type):

Cython/Compiler/TypeSlots.py

@@ -177,8 +177,8 @@ class FixedSlot(SlotDescriptor):
     #
     #  value        string
     
-    def __init__(self, slot_name, value):
-        SlotDescriptor.__init__(self, slot_name)
+    def __init__(self, slot_name, value, py3k=True):
+        SlotDescriptor.__init__(self, slot_name, py3k=py3k)
         self.value = value
     
     def slot_code(self, scope):
@@ -188,8 +188,8 @@ class FixedSlot(SlotDescriptor):
 class EmptySlot(FixedSlot):
     #  Descriptor for a type slot whose value is always 0.
     
-    def __init__(self, slot_name):
-        FixedSlot.__init__(self, slot_name, "0")
+    def __init__(self, slot_name, py3k=True):
+        FixedSlot.__init__(self, slot_name, "0", py3k=py3k)
 
 
 class MethodSlot(SlotDescriptor):
@@ -553,12 +553,12 @@ PyNumberMethods = (
     MethodSlot(binaryfunc, "nb_and", "__and__"),
     MethodSlot(binaryfunc, "nb_xor", "__xor__"),
     MethodSlot(binaryfunc, "nb_or", "__or__"),
-    EmptySlot("nb_coerce"),
+    EmptySlot("nb_coerce", py3k = False),
     MethodSlot(unaryfunc, "nb_int", "__int__"),
     MethodSlot(unaryfunc, "nb_long", "__long__"),
     MethodSlot(unaryfunc, "nb_float", "__float__"),
-    MethodSlot(unaryfunc, "nb_oct", "__oct__"),
-    MethodSlot(unaryfunc, "nb_hex", "__hex__"),
+    MethodSlot(unaryfunc, "nb_oct", "__oct__", py3k = False),
+    MethodSlot(unaryfunc, "nb_hex", "__hex__", py3k = False),
     
     # Added in release 2.0
     MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"),

Cython/Utils.py

@@ -24,7 +24,6 @@ def castrate_file(path, st):
     except EnvironmentError:
         pass
     else:
-        #st = os.stat(path)
         f.seek(0, 0)
         f.truncate()
         f.write(
@@ -33,6 +32,14 @@ def castrate_file(path, st):
         if st:
             os.utime(path, (st.st_atime, st.st_mtime))
 
+def modification_time(path):
+    st = os.stat(path)
+    return st.st_mtime
+
+def file_newer_than(path, time):
+    ftime = modification_time(path)
+    return ftime > time
+
 # support for source file encoding detection and unicode decoding
 
 def encode_filename(filename):

runtests.py

@@ -17,7 +17,7 @@ CFLAGS = os.getenv('CFLAGS', '').split()
 
 
 class ErrorWriter(object):
-    match_error = re.compile('(warning:)?(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
+    match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match
     def __init__(self):
         self.output = []
         self.write = self.output.append
@@ -31,8 +31,9 @@ class ErrorWriter(object):
                 is_warning, line, column, message = match.groups()
                 if (is_warning and collect_warnings) or \
                         (not is_warning and collect_errors):
-                    result.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
-        return result
+                    result.append( (int(line), int(column), message.strip()) )
+        result.sort()
+        return [ "%d:%d: %s" % values for values in result ]
 
     def geterrors(self):
         return self._collect(True, False)

tests/compile/extcoerce.pyx

@@ -7,7 +7,7 @@ cdef class Swallow:
     pass
     
 def f(Grail g):
-    cdef int i
+    cdef int i = 0
     cdef Swallow s
     g = x
     x = g

tests/compile/nogil.h

+void e1(void);
+void *e2(void);

tests/compile/nogil.pyx

-cdef extern void g(int x) nogil
+cdef extern object g(object x) nogil
+cdef extern void g2(object x) nogil
+
+cdef extern from "nogil.h":
+        void e1() nogil
+        int *e2() nogil
 
 cdef void f(int x) nogil:
-    cdef int y
-    y = 42
+        cdef int y
+        y = 42
+
+cdef void h(object x) nogil:
+        cdef void *p
+        g2(x)
+        g2(<object>p)
+        p = <void *>x
+        e1()
+        e2()

tests/compile/while.pyx

 def f(a, b):
-    cdef int i
+    cdef int i = 5
     
     while a:
         x = 1
@@ -23,4 +23,4 @@ def f(a, b):
         x = 1
         continue
         x = 2
-    
\ No newline at end of file
+    

tests/compile/withgil.pyx

@@ -3,3 +3,6 @@ cdef void f() with gil:
 
 cdef int g(void* x) with gil:
     pass
+
+f()
+g("test")

tests/errors/cmethbasematch.pyx

+cdef class C:
+    cdef void f(self):
+        pass
+
+cdef class D(C):
+    cdef void f(self, int x):
+        pass
+
+_ERRORS = u"""
+6: 9: Signature not compatible with previous declaration
+2: 9: Previous declaration is here
+"""

tests/errors/nogil.pyx

+cdef object f(object x) nogil:
+    pass
+
+cdef void g(int x) nogil:
+    cdef object z
+    z = None
+
+cdef void h(int x) nogil:
+    p()
+
+cdef object p() nogil:
+    pass
+
+cdef void r() nogil:
+    q()
+    
+cdef object m():
+    cdef object x, y, obj
+    cdef int i, j, k
+    global fred
+    q()
+    with nogil:
+        r()
+        q()
+        i = 42
+        obj = None
+        17L
+        7j
+        help
+        `"Hello"`
+        import fred
+        from fred import obj
+        for x in obj:
+            pass
+        obj[i]
+        obj[i:j]
+        obj[i:j:k]
+        obj.fred
+        (x, y)
+        [x, y]
+        {x: y}
+        obj and x
+        t(obj)
+#        f(42) # Cython handles this internally
+        x + obj
+        -obj
+        x = y = obj
+        x, y = y, x
+        obj[i] = x
+        obj.fred = x
+        print obj
+        del fred
+        return obj
+        raise obj
+        if obj:
+            pass
+        while obj:
+            pass
+        for x <= obj <= y:
+            pass
+        try:
+            pass
+        except:
+            pass
+        try:
+            pass
+        finally:
+            pass
+
+cdef void q():
+    pass
+
+cdef class C:
+    pass
+
+cdef void t(C c) nogil:
+    pass
+
+
+_ERRORS = u"""
+ 1: 5: Function with Python return type cannot be declared nogil
+ 6: 6: Assignment of Python object not allowed without gil
+ 4: 5: Function declared nogil has Python locals or temporaries
+ 8: 5: Function declared nogil has Python locals or temporaries
+11: 5: Function with Python return type cannot be declared nogil
+15: 5: Calling gil-requiring function without gil
+24: 9: Calling gil-requiring function without gil
+26:12: Assignment of Python object not allowed without gil
+27: 8: Constructing Python long int not allowed without gil
+28: 8: Constructing complex number not allowed without gil
+29:12: Accessing Python global or builtin not allowed without gil
+30: 8: Backquote expression not allowed without gil
+31:15: Python import not allowed without gil
+31:15: Assignment of Python object not allowed without gil
+32:13: Python import not allowed without gil
+32:25: Constructing Python list not allowed without gil
+33:17: Iterating over Python object not allowed without gil
+35:11: Indexing Python object not allowed without gil
+36:11: Slicing Python object not allowed without gil
+37:11: Constructing Python slice object not allowed without gil
+37:11: Indexing Python object not allowed without gil
+37:13: Converting to Python object not allowed without gil
+37:15: Converting to Python object not allowed without gil
+37:17: Converting to Python object not allowed without gil
+38:11: Accessing Python attribute not allowed without gil
+39: 9: Constructing Python tuple not allowed without gil
+40: 8: Constructing Python list not allowed without gil
+41: 8: Constructing Python dict not allowed without gil
+42:12: Creating temporary Python reference not allowed without gil
+42:12: Truth-testing Python object not allowed without gil
+42:17: Creating temporary Python reference not allowed without gil
+43:13: Python type test not allowed without gil
+#44: 4: Converting to Python object not allowed without gil
+45:10: Operation not allowed without gil
+46: 8: Operation not allowed without gil
+47:10: Assignment of Python object not allowed without gil
+47:14: Assignment of Python object not allowed without gil
+48: 9: Assignment of Python object not allowed without gil
+48:13: Assignment of Python object not allowed without gil
+48:16: Creating temporary Python reference not allowed without gil
+48:19: Creating temporary Python reference not allowed without gil
+49:11: Indexing Python object not allowed without gil
+49:11: Assignment of Python object not allowed without gil
+50:11: Accessing Python attribute not allowed without gil
+50:11: Assignment of Python object not allowed without gil
+51: 8: Constructing Python tuple not allowed without gil
+51: 8: Python print statement not allowed without gil
+52: 8: Deleting Python object not allowed without gil
+53: 8: Returning Python object not allowed without gil
+54: 8: Raising exception not allowed without gil
+55:14: Truth-testing Python object not allowed without gil
+57:17: Truth-testing Python object not allowed without gil
+59: 8: Converting to Python object not allowed without gil
+61: 8: Try-except statement not allowed without gil
+65: 8: Try-finally statement not allowed without gil
+"""

tests/errors/nogilfunctype.pyx

+cdef extern from *:
+    cdef void f()
+    cdef void (*fp)() nogil
+
+    cdef void g() nogil
+    cdef void (*gp)()
+
+gp = g
+
+fp = f
+
+_ERRORS = u"""
+10:6: Cannot assign type 'void (void)' to 'void (*)(void) nogil'
+"""

tests/run/big_indices.pyx

 __doc__ = u"""
     >>> test()
-    neg -1
-    pos 4294967294
+    neg False
+    pos True
     neg
     pos
     neg
@@ -12,8 +12,8 @@ def test():
     cdef long neg = -1
     cdef unsigned long pos = -2 # will be a large positive number
 
-    print "neg", neg
-    print "pos", pos
+    print "neg", neg > 0
+    print "pos", pos > -
 
     D = { neg: 'neg', pos: 'pos' }
 

tests/run/exceptionpropagation.pyx

+__doc__ = u"""
+>>> foo(0)
+>>> foo(1)
+Traceback (most recent call last):
+RuntimeError
+"""
+
+cdef int CHKERR(int ierr) except -1:
+    if ierr==0: return 0
+    raise RuntimeError
+
+cdef int obj2int(object ob) except *:
+    return ob
+
+def foo(a):
+    cdef int i = obj2int(a)
+    CHKERR(i)

tests/run/inplace.pyx

 __doc__ = u"""
     >>> f(5, 7)
-    29509034655744
+    29509034655744L
 
     >>> g(13, 4)
     32
@@ -9,6 +9,10 @@ __doc__ = u"""
     105.0
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"L", u"")
+
 def f(a,b):
     a += b
     a *= b

tests/run/modop.pyx

@@ -6,6 +6,9 @@ __doc__ = u"""
 
     >>> modint(9,2)
     1
+
+    >>> print modptr()
+    spameggs
 """
 
 def modobj(obj2, obj3):
@@ -17,11 +20,10 @@ def modint(int int2, int int3):
     int1 = int2 % int3
     return int1
 
-cdef modptr():
-    # FIXME!!!
+def modptr():
     cdef char *str2, *str3
-    str2 = "spam"
+    str2 = "spam%s"
     str3 = "eggs"
 
     obj1 = str2 % str3
-    return obj1
+    return obj1.decode(u"ASCII")

tests/compile/varargdecl.pyx → tests/run/varargdecl.pyx

+__doc__ = u"""
+>>> test()
+"""
+
 cdef grail(char *blarg, ...):
     pass
+
+def test():
+    grail("test")
+    grail("test", "toast")

tests/run/withnogil.pyx

@@ -18,5 +18,5 @@ def g():
         h()
     return 1
 
-cdef int h() except -1:
+cdef int h() nogil except -1:
     pass