Merge with cython-devel

Cython/CodeWriter.py

@@ -70,9 +70,9 @@ class CodeWriter(TreeVisitor):
         if len(items) > 0:
             for item in items[:-1]:
                 self.visit(item)
-                if output_rhs and item.rhs is not None:
+                if output_rhs and item.default is not None:
                     self.put(u" = ")
-                    self.visit(item.rhs)
+                    self.visit(item.default)
                 self.put(u", ")
             self.visit(items[-1])
     
@@ -123,12 +123,26 @@ class CodeWriter(TreeVisitor):
         self.visit(node.rhs)
         self.endline()
     
+    def visit_CascadedAssignmentNode(self, node):
+        self.startline()
+        for lhs in node.lhs_list:
+            self.visit(lhs)
+            self.put(u" = ")
+        self.visit(node.rhs)
+        self.endline()
+    
     def visit_NameNode(self, node):
         self.putname(node.name)
     
     def visit_IntNode(self, node):
         self.put(node.value)
-        
+
+    def visit_StringNode(self, node):
+        value = node.value
+        if value.encoding is not None:
+            value = value.encode(value.encoding)
+        self.put(repr(value))
+
     def visit_IfStatNode(self, node):
         # The IfClauseNode is handled directly without a seperate match
         # for clariy.

Cython/Compiler/ExprNodes.py

@@ -1677,6 +1677,17 @@ class SimpleCallNode(CallNode):
         if func_type.is_ptr:
             func_type = func_type.base_type
         return func_type
+        
+    def exception_checks(self):
+        func_type = self.function_type()
+        exc_val = func_type.exception_value
+        exc_check = func_type.exception_check
+        if exc_val is None and self.function.entry.visibility != 'extern':
+            return_type = func_type.return_type
+            if not return_type.is_struct_or_union and not return_type.is_void:
+                exc_val = return_type.cast_code(Naming.default_error)
+            exc_check = 1
+        return exc_val, exc_check
     
     def analyse_c_function_call(self, env):
         func_type = self.function_type()
@@ -1719,12 +1730,13 @@ class SimpleCallNode(CallNode):
                     "Python object cannot be passed as a varargs parameter")
         # Calc result type and code fragment
         self.type = func_type.return_type
-        if self.type.is_pyobject \
-            or func_type.exception_value is not None \
-            or func_type.exception_check:
+        if self.type.is_pyobject:
+            self.is_temp = 1
+            self.result_ctype = py_object_type
+        else:
+            exc_val, exc_check = self.exception_checks()
+            if self.type.is_pyobject or exc_val is not None or exc_check:
                 self.is_temp = 1
-                if self.type.is_pyobject:
-                    self.result_ctype = py_object_type
         # C++ exception handler
         if func_type.exception_check == '+':
             if func_type.exception_value is None:
@@ -1789,8 +1801,7 @@ class SimpleCallNode(CallNode):
             if self.type.is_pyobject:
                 exc_checks.append("!%s" % self.result_code)
             else:
-                exc_val = func_type.exception_value
-                exc_check = func_type.exception_check
+                exc_val, exc_check = self.exception_checks()
                 if exc_val is not None:
                     exc_checks.append("%s == %s" % (self.result_code, exc_val))
                 if exc_check:
@@ -3142,7 +3153,7 @@ class NumBinopNode(BinopNode):
         "+":		"PyNumber_Add",
         "-":		"PyNumber_Subtract",
         "*":		"PyNumber_Multiply",
-        "/":		"PyNumber_Divide",
+        "/":		"__Pyx_PyNumber_Divide",
         "//":		"PyNumber_FloorDivide",
         "%":		"PyNumber_Remainder",
         "**":       "PyNumber_Power"
@@ -4046,6 +4057,7 @@ class PersistentNode(ExprNode):
     subexprs = ["arg"]
     temp_counter = 0
     generate_counter = 0
+    analyse_counter = 0
     result_code = None
     
     def __init__(self, arg, uses):
@@ -4054,13 +4066,15 @@ class PersistentNode(ExprNode):
         self.uses = uses
         
     def analyse_types(self, env):
-        self.arg.analyse_types(env)
-        self.type = self.arg.type
-        self.result_ctype = self.arg.result_ctype
-        self.is_temp = 1
+        if self.analyse_counter == 0:
+            self.arg.analyse_types(env)
+            self.type = self.arg.type
+            self.result_ctype = self.arg.result_ctype
+            self.is_temp = 1
+        self.analyse_counter += 1
     
     def calculate_result_code(self):
-        return self.arg.result_code
+        return self.result_code
 
     def generate_evaluation_code(self, code):
         if self.generate_counter == 0:
@@ -4080,13 +4094,16 @@ class PersistentNode(ExprNode):
     def allocate_temps(self, env, result=None):
         if self.temp_counter == 0:
             self.arg.allocate_temps(env)
-            if result is None:
-                self.result_code = env.allocate_temp(self.type)
-            else:
-                self.result_code = result
+            self.allocate_temp(env, result)
             self.arg.release_temp(env)
         self.temp_counter += 1
         
+    def allocate_temp(self, env, result=None):
+        if result is None:
+            self.result_code = env.allocate_temp(self.type)
+        else:
+            self.result_code = result
+        
     def release_temp(self, env):
         if self.temp_counter == self.uses:
             env.release_temp(self.result_code)

Cython/Compiler/Future.py

@@ -8,5 +8,6 @@ def _get_feature(name):
 
 unicode_literals = _get_feature("unicode_literals")
 with_statement = _get_feature("with_statement")
+division = _get_feature("division")
 
 del _get_feature

Cython/Compiler/Main.py

@@ -354,6 +354,7 @@ def create_generate_code(context, options, result):
     return generate_code
 
 def create_default_pipeline(context, options, result):
+    from Visitor import PrintTree
     from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse
     from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
     from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform

Cython/Compiler/ModuleNode.py

@@ -5,6 +5,7 @@
 import os, time
 from cStringIO import StringIO
 from PyrexTypes import CPtrType
+import Future
 
 try:
     set
@@ -467,8 +468,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyInt_AsSsize_t              PyLong_AsSsize_t")
         code.putln("  #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask")
         code.putln("  #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask")
-        code.putln("  #define PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
+        code.putln("  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
         code.putln("#else")
+        if Future.division in env.context.future_directives:
+            code.putln("  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
+        else:
+            code.putln("  #define __Pyx_PyNumber_Divide(x,y)         PyNumber_Divide(x,y)")
         code.putln("  #define PyBytes_Type                 PyString_Type")
         code.putln("#endif")
 
@@ -482,6 +487,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#ifndef __cdecl")
         code.putln("  #define __cdecl")
         code.putln("#endif")
+        code.putln('');
+        code.putln('#define %s 0xB0000000B000B0BBLL' % Naming.default_error);
+        code.putln('');
         self.generate_extern_c_macro_definition(code)
         code.putln("#include <math.h>")
         code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env))

Cython/Compiler/Naming.py

@@ -74,6 +74,7 @@ import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
 cur_scope_cname  = pyrex_prefix + "cur_scope"
 enc_scope_cname  = pyrex_prefix + "enc_scope"
+default_error    = pyrex_prefix + "ERROR"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Nodes.py

@@ -177,7 +177,7 @@ class Node(object):
             if isinstance(x, Node):
                 return x.dump(level)
             elif isinstance(x, list):
-                return "[%s]" % ", ".join(dump_child(item, level) for item in x)
+                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
             else:
                 return repr(x)
             
@@ -357,11 +357,13 @@ class CDeclaratorNode(Node):
 
 
 class CNameDeclaratorNode(CDeclaratorNode):
-    #  name   string             The Pyrex name being declared
-    #  cname  string or None     C name, if specified
-    #  rhs    ExprNode or None   the value assigned on declaration
+    #  name    string             The Pyrex name being declared
+    #  cname   string or None     C name, if specified
+    #  default ExprNode or None   the value assigned on declaration
     
-    child_attrs = []
+    child_attrs = ['default']
+    
+    default = None
     
     def analyse(self, base_type, env, nonempty = 0):
         if nonempty and self.name == '':
@@ -373,25 +375,25 @@ class CNameDeclaratorNode(CDeclaratorNode):
         
     def analyse_expressions(self, env):
         self.entry = env.lookup(self.name)
-        if self.rhs is not None:
-            env.control_flow.set_state(self.rhs.end_pos(), (self.entry.name, 'initalized'), True)
-            env.control_flow.set_state(self.rhs.end_pos(), (self.entry.name, 'source'), 'assignment')
+        if self.default is not None:
+            env.control_flow.set_state(self.default.end_pos(), (self.entry.name, 'initalized'), True)
+            env.control_flow.set_state(self.default.end_pos(), (self.entry.name, 'source'), 'assignment')
             self.entry.used = 1
             if self.type.is_pyobject:
                 self.entry.init_to_none = False
                 self.entry.init = 0
-            self.rhs.analyse_types(env)
-            self.rhs = self.rhs.coerce_to(self.type, env)
-            self.rhs.allocate_temps(env)
-            self.rhs.release_temp(env)
+            self.default.analyse_types(env)
+            self.default = self.default.coerce_to(self.type, env)
+            self.default.allocate_temps(env)
+            self.default.release_temp(env)
 
     def generate_execution_code(self, code):
-        if self.rhs is not None:
-            self.rhs.generate_evaluation_code(code)
+        if self.default is not None:
+            self.default.generate_evaluation_code(code)
             if self.type.is_pyobject:
-                self.rhs.make_owned_reference(code)
-            code.putln('%s = %s;' % (self.entry.cname, self.rhs.result_as(self.entry.type)))
-            self.rhs.generate_post_assignment_code(code)
+                self.default.make_owned_reference(code)
+            code.putln('%s = %s;' % (self.entry.cname, self.default.result_as(self.entry.type)))
+            self.default.generate_post_assignment_code(code)
             code.putln()
 
 class CPtrDeclaratorNode(CDeclaratorNode):
@@ -1211,11 +1213,18 @@ class CFuncDefNode(FuncDefNode):
         if self.return_type.is_pyobject:
             return "0"
         else:
-            #return None
-            return self.entry.type.exception_value
+            if self.entry.type.exception_value is not None:
+                return self.entry.type.exception_value
+            elif self.return_type.is_struct_or_union or self.return_type.is_void:
+                return None
+            else:
+                return self.return_type.cast_code(Naming.default_error)
             
     def caller_will_check_exceptions(self):
-        return self.entry.type.exception_check
+        if self.entry.type.exception_value is None:
+            return 1
+        else:
+            return self.entry.type.exception_check
                     
     def generate_optarg_wrapper_function(self, env, code):
         if self.type.optional_arg_count and \
@@ -1917,9 +1926,7 @@ class OverrideCheckNode(StatNode):
                                              function=self.func_node, 
                                              args=[ExprNodes.NameNode(self.pos, name=arg.name) for arg in self.args[first_arg:]])
         self.body = ReturnStatNode(self.pos, value=call_node)
-#        self.func_temp = env.allocate_temp_pyobject()
         self.body.analyse_expressions(env)
-#        env.release_temp(self.func_temp)
         
     def generate_execution_code(self, code):
         # Check to see if we are an extension type
@@ -1934,7 +1941,7 @@ class OverrideCheckNode(StatNode):
             code.putln("else {")
         else:
             code.putln("else if (unlikely(Py_TYPE(%s)->tp_dictoffset != 0)) {" % self_arg)
-        err = code.error_goto_if_null(self_arg, self.pos)
+        err = code.error_goto_if_null(self.func_node.result_code, self.pos)
         # need to get attribute manually--scope would return cdef method
         code.putln("%s = PyObject_GetAttr(%s, %s); %s" % (self.func_node.result_code, self_arg, self.py_func.interned_attr_cname, err))
         # It appears that this type is not anywhere exposed in the Python/C API
@@ -1943,7 +1950,7 @@ class OverrideCheckNode(StatNode):
         code.putln('if (!%s || %s) {' % (is_builtin_function_or_method, is_overridden))
         self.body.generate_execution_code(code)
         code.putln('}')
-#        code.put_decref(self.func_temp, PyrexTypes.py_object_type)
+        code.put_decref_clear(self.func_node.result_code, PyrexTypes.py_object_type)
         code.putln("}")
 
 class ClassDefNode(StatNode, BlockNode):

Cython/Compiler/Optimize.py

@@ -71,6 +71,8 @@ class SwitchTransform(Visitor.VisitorTransform):
                 cases.append(Nodes.SwitchCaseNode(pos = if_clause.pos,
                                                   conditions = conditions,
                                                   body = if_clause.body))
+        
+        common_var = unwrap_node(common_var)
         return Nodes.SwitchStatNode(pos = node.pos,
                                     test = common_var,
                                     cases = cases,

Cython/Compiler/ParseTreeTransforms.py

@@ -72,11 +72,6 @@ class NormalizeTree(CythonTransform):
         else:
             return []
 
-    # Eliminate CascadedAssignmentNode
-    def visit_CascadedAssignmentNode(self, node):
-        tmpname = temp_name_handle()
-        
-
 
 class PostParseError(CompileError): pass
 

Cython/Compiler/Parsing.py

@@ -8,7 +8,7 @@ from Scanning import PyrexScanner, FileSourceDescriptor
 import Nodes
 import ExprNodes
 from ModuleNode import ModuleNode
-from Errors import error, InternalError
+from Errors import error, warning, InternalError
 from Cython import Utils
 import Future
 
@@ -1784,7 +1784,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
             name = ""
             cname = None
         result = Nodes.CNameDeclaratorNode(pos,
-            name = name, cname = cname, rhs = rhs)
+            name = name, cname = cname, default = rhs)
     result.calling_convention = calling_convention
     return result
 
@@ -2281,9 +2281,14 @@ def p_property_decl(s):
 
 def p_doc_string(s):
     if s.sy == 'BEGIN_STRING':
-        _, result = p_cat_string_literal(s)
+        pos = s.position()
+        kind, result = p_cat_string_literal(s)
         if s.sy != 'EOF':
             s.expect_newline("Syntax error in doc string")
+        if kind != 'u':
+            # warning(pos, "Python 3 requires docstrings to be unicode strings")
+            if kind == 'b':
+                result.encoding = None # force a unicode string
         return result
     else:
         return None

Cython/Compiler/Visitor.py

@@ -213,6 +213,7 @@ class PrintTree(TreeVisitor):
     def __call__(self, tree, phase=None):
         print("Parse tree dump at phase '%s'" % phase)
         self.visit(tree)
+        return tree
 
     # Don't do anything about process_list, the defaults gives
     # nice-looking name[idx] nodes which will visually appear

bin/update_references

-#!/usr/bin/env python
-#
-#   Go through the Tests directory and its subdirectories
-#   copying the latest versions of the test outputs into
-#   the Reference directories.
-#
-
-import os, sys
-
-ignore_names = [".DS_Store", "Icon\r"]
-
-def copy_file(from_path, to_path):
-	# We copy the contents from one file to the other
-	# so as to preserve metadata on the Mac.
-	#print from_path, "-->", to_path
-	f = open(from_path)
-	g = open(to_path, "w+")
-	g.write(f.read())
-	f.close()
-	g.close()
-
-def update_references(out_dir, ref_dir):
-	for name in os.listdir(ref_dir):
-		if name not in ignore_names:
-			out_file = os.path.join(out_dir, name)
-			ref_file = os.path.join(ref_dir, name)
-			if os.path.isfile(out_file):
-				print "Updating", name
-				copy_file(out_file, ref_file)
-
-def update_references_in_dir(dir):
-	print "Updating references in", dir
-	for name in os.listdir(dir):
-		if name <> "Reference" and not name.startswith("("):
-			item_path = os.path.join(dir, name)
-			if os.path.isdir(item_path):
-				update_references_in_dir(item_path)
-	ref_dir = os.path.join(dir, "Reference")
-	if os.path.isdir(ref_dir):
-		update_references(dir, ref_dir)
-
-def main():
-	bin_dir = os.path.dirname(sys.argv[0])
-	source_dir = os.path.dirname(bin_dir)
-	tests_dir = os.path.join(source_dir, "Tests")
-	update_references_in_dir(tests_dir)
-
-if __name__ == "__main__":
-	main()

runtests.py

@@ -45,13 +45,15 @@ class ErrorWriter(object):
         return self._collect(True, True)
 
 class TestBuilder(object):
-    def __init__(self, rootdir, workdir, selectors, annotate, cleanup_workdir, cleanup_sharedlibs):
+    def __init__(self, rootdir, workdir, selectors, annotate,
+                 cleanup_workdir, cleanup_sharedlibs, with_pyregr):
         self.rootdir = rootdir
         self.workdir = workdir
         self.selectors = selectors
         self.annotate = annotate
         self.cleanup_workdir = cleanup_workdir
         self.cleanup_sharedlibs = cleanup_sharedlibs
+        self.with_pyregr = with_pyregr
 
     def build_suite(self):
         suite = unittest.TestSuite()
@@ -63,6 +65,8 @@ class TestBuilder(object):
                 continue
             path = os.path.join(self.rootdir, filename)
             if os.path.isdir(path) and filename in TEST_DIRS:
+                if filename == 'pyregr' and not self.with_pyregr:
+                    continue
                 suite.addTest(
                     self.handle_directory(path, filename))
         return suite
@@ -117,6 +121,7 @@ class CythonCompileTestCase(unittest.TestCase):
         self.expect_errors = expect_errors
         self.annotate = annotate
         self.cleanup_workdir = cleanup_workdir
+        self.cleanup_sharedlibs = cleanup_sharedlibs
         unittest.TestCase.__init__(self)
 
     def shortDescription(self):
@@ -301,7 +306,7 @@ def collect_unittests(path, suite, selectors):
                     module = __import__(modulename)
                     for x in modulename.split('.')[1:]:
                         module = getattr(module, x)
-                    suite.addTests(loader.loadTestsFromModule(module))
+                    suite.addTests([loader.loadTestsFromModule(module)])
 
 if __name__ == '__main__':
     from optparse import OptionParser
@@ -321,6 +326,9 @@ if __name__ == '__main__':
     parser.add_option("--no-file", dest="filetests",
                       action="store_false", default=True,
                       help="do not run the file based tests")
+    parser.add_option("--no-pyregr", dest="pyregr",
+                      action="store_false", default=True,
+                      help="do not run the regression tests of CPython in tests/pyregr/")
     parser.add_option("-C", "--coverage", dest="coverage",
                       action="store_true", default=False,
                       help="collect source coverage data for the Compiler")
@@ -378,8 +386,8 @@ if __name__ == '__main__':
     if options.filetests:
         filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
                                 options.annotate_source, options.cleanup_workdir,
-                                options.cleanup_sharedlibs)
-        test_suite.addTests(filetests.build_suite())
+                                options.cleanup_sharedlibs, options.pyregr)
+        test_suite.addTests([filetests.build_suite()])
 
     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 

tests/run/future_division.pyx

+from __future__ import division
+
+__doc__ = """
+>>> from future_division import doit
+>>> doit(1,2)
+(0.5, 0)
+>>> doit(4,3)
+(1.3333333333333333, 1)
+>>> doit(4,3.0)
+(1.3333333333333333, 1.0)
+>>> doit(4,2)
+(2.0, 2)
+"""
+
+def doit(x,y):
+    return x/y, x//y

tests/run/switch.pyx

@@ -25,9 +25,9 @@ __doc__ = u"""
 >>> switch_py(8)
 4
 >>> switch_py(10)
-7
+10
 >>> switch_py(12)
-8
+12
 >>> switch_py(13)
 0
 
@@ -57,9 +57,9 @@ __doc__ = u"""
 >>> switch_c(8)
 4
 >>> switch_c(10)
-7
+10
 >>> switch_c(12)
-8
+12
 >>> switch_c(13)
 0
 """
@@ -86,10 +86,10 @@ def switch_py(x):
         return 3
     elif x in [4,5,7,8]:
         return 4
-    elif x in (10,11): # doesn't work:  (7,8,10,11)
-        return 7
+    elif x in (10,11):
+        return 10
     elif x in (12,):
-        return 8
+        return 12
     else:
         return 0
     return -1
@@ -116,10 +116,10 @@ def switch_c(int x):
         return 3
     elif x in [4,5,7,8]:
         return 4
-    elif x in (10,11): # doesn't work:  (7,8,10,11)
-        return 7
+    elif x in (10,11):
+        return 10
     elif x in (12,):
-        return 8
+        return 12
     else:
         return 0
     return -1