merge

Cython/Compiler/PyrexTypes.py

@@ -2,7 +2,6 @@
 #   Pyrex - Types
 #
 
-import string
 import Naming
 
 class BaseType:
@@ -103,7 +102,7 @@ class PyrexType(BaseType):
         return str(value)
     
     def __str__(self):
-        return string.strip(self.declaration_code("", for_display = 1))
+        return self.declaration_code("", for_display = 1).strip()
     
     def same_as(self, other_type, **kwds):
         return self.same_as_resolved_type(other_type.resolve(), **kwds)
@@ -655,7 +654,7 @@ class CFuncType(CType):
         return "<CFuncType %s %s[%s]>" % (
             repr(self.return_type),
             self.calling_convention_prefix(),
-            string.join(arg_reprs, ","))
+            ",".join(arg_reprs))
     
     def calling_convention_prefix(self):
         cc = self.calling_convention
@@ -786,7 +785,7 @@ class CFuncType(CType):
             arg_decl_list.append(self.op_arg_struct.declaration_code(Naming.optional_args_cname))
         if self.has_varargs:
             arg_decl_list.append("...")
-        arg_decl_code = string.join(arg_decl_list, ", ")
+        arg_decl_code = ", ".join(arg_decl_list)
         if not arg_decl_code and not pyrex:
             arg_decl_code = "void"
         exc_clause = ""
@@ -924,6 +923,7 @@ class CEnumType(CType):
 
 
 def _escape_byte_string(s):
+    s = s.replace('\0', r'\x00')
     try:
         s.decode("ASCII")
         return s

runtests.py

 #!/usr/bin/python
 
-import os, sys, unittest, doctest
+import os, sys, re, shutil, unittest, doctest
 
-from Cython.Distutils.extension import Extension
-from Cython.Distutils import build_ext
+from Cython.Compiler.Main import \
+    CompilationOptions, \
+    default_options as pyrex_default_options, \
+    compile as cython_compile
 
 from distutils.dist import Distribution
+from distutils.core import Extension
+from distutils.command.build_ext import build_ext
 distutils_distro = Distribution()
 
-TEST_DIRS = ['compile', 'run']
+TEST_DIRS = ['compile', 'errors', 'run']
 TEST_RUN_DIRS = ['run']
 
 INCLUDE_DIRS = [ d for d in os.getenv('INCLUDE', '').split(os.pathsep) if d ]
 CFLAGS = os.getenv('CFLAGS', '').split()
 
+
+class ErrorWriter(object):
+    match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
+    def __init__(self):
+        self.output = []
+        self.write = self.output.append
+
+    def geterrors(self):
+        s = ''.join(self.output)
+        errors = []
+        for line in s.split('\n'):
+            match = self.match_error(line)
+            if match:
+                line, column, message = match.groups()
+                errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
+        return errors
+
 class TestBuilder(object):
     def __init__(self, rootdir, workdir, selectors):
         self.rootdir = rootdir
@@ -30,6 +51,7 @@ class TestBuilder(object):
         return suite
 
     def handle_directory(self, path, context):
+        expect_errors = (context == 'errors')
         suite = unittest.TestSuite()
         for filename in os.listdir(path):
             if not filename.endswith(".pyx"):
@@ -39,66 +61,124 @@ class TestBuilder(object):
             if not [ 1 for match in self.selectors
                      if match(fqmodule) ]:
                 continue
-            suite.addTest(
-                CythonCompileTestCase(path, self.workdir, module))
             if context in TEST_RUN_DIRS:
-                suite.addTest(
-                    CythonRunTestCase(self.workdir, module))
+                test = CythonRunTestCase(path, self.workdir, module)
+            else:
+                test = CythonCompileTestCase(
+                    path, self.workdir, module, expect_errors)
+            suite.addTest(test)
         return suite
 
 class CythonCompileTestCase(unittest.TestCase):
-    def __init__(self, directory, workdir, module):
+    def __init__(self, directory, workdir, module, expect_errors=False):
         self.directory = directory
         self.workdir = workdir
         self.module = module
+        self.expect_errors = expect_errors
         unittest.TestCase.__init__(self)
 
     def shortDescription(self):
         return "compiling " + self.module
 
-    def runTest(self):
-        self.compile(self.directory, self.module, self.workdir)
+    def setUp(self):
+        if os.path.exists(self.workdir):
+            shutil.rmtree(self.workdir, ignore_errors=True)
+        os.makedirs(self.workdir)
 
-    def compile(self, directory, module, workdir):
+    def runTest(self):
+        self.compile(self.directory, self.module, self.workdir,
+                     self.directory, self.expect_errors)
+
+    def split_source_and_output(self, directory, module, workdir):
+        source_and_output = open(os.path.join(directory, module + '.pyx'), 'rU')
+        out = open(os.path.join(workdir, module + '.pyx'), 'w')
+        for line in source_and_output:
+            last_line = line
+            if line.startswith("_ERRORS"):
+                out.close()
+                out = ErrorWriter()
+            else:
+                out.write(line)
+        try:
+            geterrors = out.geterrors
+        except AttributeError:
+            return []
+        else:
+            return geterrors()
+
+    def run_cython(self, directory, module, targetdir, incdir):
+        include_dirs = INCLUDE_DIRS[:]
+        if incdir:
+            include_dirs.append(incdir)
+        source = os.path.join(directory, module + '.pyx')
+        target = os.path.join(targetdir, module + '.c')
+        options = CompilationOptions(
+            pyrex_default_options,
+            include_path = include_dirs,
+            output_file = target,
+            use_listing_file = False, cplus = False, generate_pxi = False)
+        cython_compile(source, options=options,
+                       full_module_name=module)
+
+    def run_distutils(self, module, workdir, incdir):
+        cwd = os.getcwd()
+        os.chdir(workdir)
+        try:
             build_extension = build_ext(distutils_distro)
             build_extension.include_dirs = INCLUDE_DIRS[:]
-        build_extension.include_dirs.append(directory)
+            if incdir:
+                build_extension.include_dirs.append(incdir)
             build_extension.finalize_options()
 
             extension = Extension(
                 module,
-            sources = [os.path.join(directory, module + '.pyx')],
+                sources = [module + '.c'],
                 extra_compile_args = CFLAGS,
-            pyrex_c_in_temp = 1
                 )
             build_extension.extensions = [extension]
             build_extension.build_temp = workdir
             build_extension.build_lib  = workdir
-        build_extension.pyrex_c_in_temp = 1
             build_extension.run()
+        finally:
+            os.chdir(cwd)
 
-class CythonRunTestCase(unittest.TestCase):
-    def __init__(self, rootdir, module):
-        self.rootdir, self.module = rootdir, module
-        unittest.TestCase.__init__(self)
+    def compile(self, directory, module, workdir, incdir, expect_errors):
+        expected_errors = errors = ()
+        if expect_errors:
+            expected_errors = self.split_source_and_output(
+                directory, module, workdir)
+            directory = workdir
 
-    def shortDescription(self):
-        return "running " + self.module
+        old_stderr = sys.stderr
+        try:
+            sys.stderr = ErrorWriter()
+            self.run_cython(directory, module, workdir, incdir)
+            errors = sys.stderr.geterrors()
+        finally:
+            sys.stderr = old_stderr
+
+        if errors or expected_errors:
+            for expected, error in zip(expected_errors, errors):
+                self.assertEquals(expected, error)
+            if len(errors) < len(expected_errors):
+                expected_error = expected_errors[len(errors)]
+                self.assertEquals(expected_error, None)
+            elif len(errors) > len(expected_errors):
+                unexpected_error = errors[len(expected_errors)]
+                self.assertEquals(None, unexpected_error)
+        else:
+            self.run_distutils(module, workdir, incdir)
 
-    def runTest(self):
-        self.run()
+class CythonRunTestCase(CythonCompileTestCase):
+    def shortDescription(self):
+        return "compiling and running " + self.module
 
     def run(self, result=None):
-        if not sys.path or sys.path[0] != self.rootdir:
-            sys.path.insert(0, self.rootdir)
-        if result is None: result = self.defaultTestResult()
-        try:
+        if result is None:
+            result = self.defaultTestResult()
         try:
+            self.runTest()
             doctest.DocTestSuite(self.module).run(result)
-            except ImportError:
-                result.startTest(self)
-                result.addFailure(self, sys.exc_info())
-                result.stopTest(self)
         except Exception:
             result.startTest(self)
             result.addError(self, sys.exc_info())
@@ -111,6 +191,9 @@ if __name__ == '__main__':
     if not os.path.exists(WORKDIR):
         os.makedirs(WORKDIR)
 
+    if not sys.path or sys.path[0] != WORKDIR:
+        sys.path.insert(0, WORKDIR)
+
     try:
         sys.argv.remove("-C")
     except ValueError:

tests/errors/undefinedname.pyx

+i = _this_global_name_does_not_exist_
+
+def test(i):
+    return _this_local_name_does_not_exist_
+
+_ERRORS = """
+1:37:undeclared name not builtin: _this_global_name_does_not_exist_
+4:43:undeclared name not builtin: _this_local_name_does_not_exist_
+"""

tests/run/append.pyx

@@ -4,7 +4,7 @@ None
 None
 got error
 [1, 2]
->>> test_append(A())
+>>> test_append(A()) # doctest: +ELLIPSIS
 appending
 1
 appending