Merge

Cython/CodeWriter.py

@@ -149,8 +149,8 @@ class CodeWriter(TreeVisitor):
     
     def visit_PrintStatNode(self, node):
         self.startline(u"print ")
-        self.comma_seperated_list(node.args)
-        if node.ends_with_comma:
+        self.comma_seperated_list(node.arg_tuple.args)
+        if not node.append_newline:
             self.put(u",")
         self.endline()
 

Cython/Compiler/Builtin.py

@@ -2,8 +2,9 @@
 #   Pyrex - Builtin Definitions
 #
 
-from Symtab import BuiltinScope
+from Symtab import BuiltinScope, StructOrUnionScope
 from TypeSlots import Signature
+import PyrexTypes
 
 builtin_function_table = [
     # name,        args,   return,  C API func,           py equiv = "*"
@@ -24,7 +25,7 @@ builtin_function_table = [
     #('hex',       "",     "",      ""),
     #('id',        "",     "",      ""),
     #('input',     "",     "",      ""),
-    ('intern',     "s",    "O",     "PyString_InternFromString"),
+    ('intern',     "s",    "O",     "__Pyx_InternFromString"),
     ('isinstance', "OO",   "b",     "PyObject_IsInstance"),
     ('issubclass', "OO",   "b",     "PyObject_IsSubclass"),
     ('iter',       "O",    "O",     "PyObject_GetIter"),
@@ -100,6 +101,21 @@ builtin_types_table = [
                                     ("values","O",   "O", "PyDict_Values")]),
 ]
 
+builtin_structs_table = [
+    ('Py_buffer', 'Py_buffer',
+     [("buf",        PyrexTypes.c_void_ptr_type),
+      ("len",        PyrexTypes.c_py_ssize_t_type),
+      ("readonly",   PyrexTypes.c_bint_type),
+      ("format",     PyrexTypes.c_char_ptr_type),
+      ("ndim",       PyrexTypes.c_int_type),
+      ("shape",      PyrexTypes.c_py_ssize_t_ptr_type),
+      ("strides",    PyrexTypes.c_py_ssize_t_ptr_type),
+      ("suboffsets", PyrexTypes.c_py_ssize_t_ptr_type),
+      ("itemsize",   PyrexTypes.c_py_ssize_t_type),
+      ("internal",   PyrexTypes.c_void_ptr_type),
+      ])
+]
+
 getattr3_utility_code = ["""
 static PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/
 ""","""
@@ -118,8 +134,18 @@ bad:
 }
 """]
 
+intern_utility_code = ["""
+#if PY_MAJOR_VERSION >= 3
+#  define __Pyx_InternFromString(s) PyUnicode_InternFromString(s)
+#else
+#  define __Pyx_InternFromString(s) PyString_InternFromString(s)
+#endif
+""","""
+"""]
+
 builtin_utility_code = {
     'getattr3': getattr3_utility_code,
+    'intern'  : intern_utility_code,
 }
 
 builtin_scope = BuiltinScope()
@@ -141,9 +167,19 @@ def init_builtin_types():
             sig = Signature(args, ret)
             the_type.scope.declare_cfunction(name, sig.function_type(), None, cname)
 
+def init_builtin_structs():
+    for name, cname, attribute_types in builtin_structs_table:
+        scope = StructOrUnionScope(name)
+        for attribute_name, attribute_type in attribute_types:
+            scope.declare_var(
+                attribute_name, attribute_type, None, attribute_name)
+        builtin_scope.declare_struct_or_union(
+            name, "struct", scope, 1, None, cname = cname)
+
 def init_builtins():
     init_builtin_funcs()
     init_builtin_types()
+    init_builtin_structs()
     global list_type, tuple_type, dict_type
     list_type  = builtin_scope.lookup('list').type
     tuple_type = builtin_scope.lookup('tuple').type

Cython/Compiler/Future.py

+def _get_feature(name):
+    import __future__
+    try:
+        return getattr(__future__, name)
+    except AttributeError:
+        # unique fake object for earlier Python versions
+        return object()
+
+unicode_literals = _get_feature("unicode_literals")
+
+del _get_feature

Cython/Compiler/Lexicon.py

@@ -5,7 +5,8 @@
 #   to be rebuilt next time pyrexc is run.
 #
 
-string_prefixes = "cCrRuU"
+raw_prefixes = "rR"
+string_prefixes = "cCuUbB"
 
 def make_lexicon():
     from Cython.Plex import \
@@ -55,9 +56,8 @@ def make_lexicon():
         + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) 
         + Str('"""')
     )
-    stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string)
     
-    beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+    beginstring = Opt(Any(string_prefixes)) + Opt(Any(raw_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
     two_oct = octdigit + octdigit
     three_oct = octdigit + octdigit + octdigit
     two_hex = hexdigit + hexdigit

Cython/Compiler/Main.py

@@ -7,6 +7,12 @@ if sys.version_info[:2] < (2, 2):
     sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n")
     sys.exit(1)
 
+try:
+    set
+except NameError:
+    # Python 2.3
+    from sets import Set as set
+
 from time import time
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
@@ -45,12 +51,14 @@ class Context:
     #
     #  modules               {string : ModuleScope}
     #  include_directories   [string]
+    #  future_directives     [object]
     
     def __init__(self, include_directories):
         #self.modules = {"__builtin__" : BuiltinScope()}
         import Builtin
         self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.include_directories = include_directories
+        self.future_directives = set()
         
     def find_module(self, module_name, 
             relative_to = None, pos = None, need_pxd = 1):

Cython/Compiler/Naming.py

@@ -62,6 +62,8 @@ c_api_tab_cname  = pyrex_prefix + "c_api_tab"
 gilstate_cname   = pyrex_prefix + "state"
 skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
 empty_tuple      = pyrex_prefix + "empty_tuple"
+print_function   = pyrex_prefix + "print"
+print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
 optional_args_cname = pyrex_prefix + "optional_args"
 no_opt_args      = pyrex_prefix + "no_opt_args"

Cython/Compiler/Options.py

@@ -2,7 +2,6 @@
 #  Pyrex - Compilation-wide options
 #
 
-intern_names = 1    #  Intern global variable and attribute names
 cache_builtins = 1  #  Perform lookups on builtin names only once
 
 embed_pos_in_docstring = 0

Cython/Compiler/PyrexTypes.py

@@ -2,6 +2,7 @@
 #   Pyrex - Types
 #
 
+from Cython import Utils
 import Naming
 
 class BaseType:
@@ -249,7 +250,7 @@ class BuiltinObjectType(PyObjectType):
         return type.is_pyobject and self.assignable_from(type)
         
     def type_test_code(self, arg):
-        return 'likely(Py%s_CheckExact(%s)) || (%s) == Py_None || (PyErr_Format(PyExc_TypeError, "Expected %s, got %%s", %s->ob_type->tp_name), 0)' % (self.name[0].upper() + self.name[1:], arg, arg, self.name, arg)
+        return 'likely(Py%s_CheckExact(%s)) || (%s) == Py_None || (PyErr_Format(PyExc_TypeError, "Expected %s, got %%s", Py_TYPE(%s)->tp_name), 0)' % (self.name[0].upper() + self.name[1:], arg, arg, self.name, arg)
 
 
 class PyExtensionType(PyObjectType):
@@ -922,23 +923,6 @@ class CEnumType(CType):
             return self.base_declaration_code(public_decl(base, dll_linkage), entity_code)
 
 
-def _escape_byte_string(s):
-    s = s.replace('\0', r'\x00')
-    try:
-        s.decode("ASCII")
-        return s
-    except UnicodeDecodeError:
-        pass
-    l = []
-    append = l.append
-    for c in s:
-        o = ord(c)
-        if o >= 128:
-            append('\\x%X' % o)
-        else:
-            append(c)
-    return ''.join(l)
-
 class CStringType:
     #  Mixin class for C string types.
 
@@ -951,7 +935,7 @@ class CStringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CUTF8StringType:
@@ -965,7 +949,7 @@ class CUTF8StringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CCharArrayType(CStringType, CArrayType):
@@ -998,16 +982,6 @@ class CCharPtrType(CStringType, CPtrType):
         CPtrType.__init__(self, c_char_type)
 
 
-class CUTF8CharPtrType(CUTF8StringType, CPtrType):
-    # C 'char *' type, encoded in UTF-8.
-    
-    parsetuple_format = "s"
-    pymemberdef_typecode = "T_STRING"
-    
-    def __init__(self):
-        CPtrType.__init__(self, c_char_type)
-
-
 class ErrorType(PyrexType):
     # Used to prevent propagation of error messages.
     
@@ -1074,14 +1048,17 @@ c_null_ptr_type =     CNullPtrType(c_void_type)
 c_char_array_type =   CCharArrayType(None)
 c_utf8_char_array_type =   CUTF8CharArrayType(None)
 c_char_ptr_type =     CCharPtrType()
-c_utf8_char_ptr_type =     CUTF8CharPtrType()
 c_char_ptr_ptr_type = CPtrType(c_char_ptr_type)
+c_py_ssize_t_ptr_type =  CPtrType(c_py_ssize_t_type)
 c_int_ptr_type =      CPtrType(c_int_type)
 
 c_returncode_type =   CIntType(2, 1, "T_INT", is_returncode = 1)
 
 c_anon_enum_type =    CAnonEnumType(-1, 1)
 
+# the Py_buffer type is defined in Builtin.py
+c_py_buffer_ptr_type = CPtrType(CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer"))
+
 error_type =    ErrorType()
 
 lowest_float_rank = 6

Cython/Compiler/Scanning.py

@@ -11,11 +11,11 @@ import stat
 import sys
 from time import time
 
-from Cython import Plex
+from Cython import Plex, Utils
 from Cython.Plex import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
-from Lexicon import string_prefixes, make_lexicon
+from Lexicon import string_prefixes, raw_prefixes, make_lexicon
 
 from Cython import Utils
 
@@ -322,6 +322,8 @@ class PyrexScanner(Scanner):
     def begin_string_action(self, text):
         if text[:1] in string_prefixes:
             text = text[1:]
+        if text[:1] in raw_prefixes:
+            text = text[1:]
         self.begin(self.string_states[text])
         self.produce('BEGIN_STRING')
     
@@ -380,8 +382,12 @@ class PyrexScanner(Scanner):
             sy, systring = self.read()
         except UnrecognizedInput:
             self.error("Unrecognized character")
-        if sy == 'IDENT' and systring in self.resword_dict:
+        if sy == 'IDENT':
+            if systring in self.resword_dict:
                 sy = systring
+            else:
+                systring = Utils.EncodedString(systring)
+                systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
         if debug_scanner:

Cython/Compiler/TypeSlots.py

@@ -3,6 +3,7 @@
 #           and associated know-how.
 #
 
+from Cython import Utils
 import Naming
 import PyrexTypes
 import sys
@@ -25,12 +26,14 @@ class Signature:
     #    'p'  void *
     #    'P'  void **
     #    'i'  int
+    #    'b'  bint
     #    'I'  int *
     #    'l'  long
     #    'Z'  Py_ssize_t
     #    's'  char *
     #    'S'  char **
     #    'r'  int used only to signal exception
+    #    'B'  Py_buffer *
     #    '-'  dummy 'self' argument (not used)
     #    '*'  rest of args passed as generic Python
     #           arg tuple and kw dict (must be last
@@ -49,6 +52,7 @@ class Signature:
         's': PyrexTypes.c_char_ptr_type,
         'S': PyrexTypes.c_char_ptr_ptr_type,
         'r': PyrexTypes.c_returncode_type,
+        'B': PyrexTypes.c_py_buffer_ptr_type,
         # 'T', '-' and '*' are handled otherwise
         # and are not looked up in here
     }
@@ -121,11 +125,15 @@ class SlotDescriptor:
     #  slot_name    string           Member name of the slot in the type object
     #  is_initialised_dynamically    Is initialised by code in the module init function
     #  flag                          Py_TPFLAGS_XXX value indicating presence of slot
+    #  py3k                          Indicates presence of slot in Python 3
+    #  py2                           Indicates presence of slot in Python 2
 
-    def __init__(self, slot_name, dynamic = 0, flag = None):
+    def __init__(self, slot_name, dynamic = 0, flag = None, py3k = True, py2 = True):
         self.slot_name = slot_name
         self.is_initialised_dynamically = dynamic
         self.flag = flag
+        self.py3k = py3k
+        self.py2  = py2
 
     def generate(self, scope, code):
         if self.is_initialised_dynamically:
@@ -133,11 +141,19 @@ class SlotDescriptor:
         else:
             value = self.slot_code(scope)
         flag = self.flag
+        py3k = self.py3k
+        py2  = self.py2
+        if not py3k:
+            code.putln("#if PY_MAJOR_VERSION < 3")
+        elif not py2:
+            code.putln("#if PY_MAJOR_VERSION >= 3")
         if flag:
-            code.putln("#if Py_TPFLAGS_DEFAULT & %s" % flag)
+            code.putln("#if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & %s)" % flag)
         code.putln("%s, /*%s*/" % (value, self.slot_name))
         if flag:
             code.putln("#endif")
+        if not py3k or not py2:
+            code.putln("#endif")
 
     # Some C implementations have trouble statically 
     # initialising a global with a pointer to an extern 
@@ -183,8 +199,8 @@ class MethodSlot(SlotDescriptor):
     #  method_name  string           The __xxx__ name of the method
     #  default      string or None   Default value of the slot
     
-    def __init__(self, signature, slot_name, method_name, default = None, flag = None):
-        SlotDescriptor.__init__(self, slot_name, flag = flag)
+    def __init__(self, signature, slot_name, method_name, default = None, flag = None, py3k=True, py2=True):
+        SlotDescriptor.__init__(self, slot_name, flag = flag, py3k = py3k, py2=py2)
         self.signature = signature
         self.slot_name = slot_name
         self.method_name = method_name
@@ -205,8 +221,8 @@ class InternalMethodSlot(SlotDescriptor):
     #
     #  slot_name    string           Member name of the slot in the type object
 
-    def __init__(self, slot_name):
-        SlotDescriptor.__init__(self, slot_name)
+    def __init__(self, slot_name, py2 = True):
+        SlotDescriptor.__init__(self, slot_name, py2 = py2)
 
     def slot_code(self, scope):
         return scope.mangle_internal(self.slot_name)
@@ -264,8 +280,8 @@ class SyntheticSlot(InternalMethodSlot):
     #  alternative default value will be placed in the type
     #  slot.
     
-    def __init__(self, slot_name, user_methods, default_value):
-        InternalMethodSlot.__init__(self, slot_name)
+    def __init__(self, slot_name, user_methods, default_value, py2 = True):
+        InternalMethodSlot.__init__(self, slot_name, py2 = py2)
         self.user_methods = user_methods
         self.default_value = default_value
     
@@ -291,7 +307,11 @@ class DocStringSlot(SlotDescriptor):
     
     def slot_code(self, scope):
         if scope.doc is not None:
-            return '"%s"' % scope.doc
+            if scope.doc.is_unicode:
+                doc = scope.doc.utf8encode()
+            else:
+                doc = scope.doc.byteencode()
+            return '"%s"' % Utils.escape_byte_string(doc)
         else:
             return "0"
 
@@ -492,6 +512,10 @@ initproc = Signature("T*", 'r')            # typedef int (*initproc)(PyObject *,
                                            # typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
                                            # typedef PyObject *(*allocfunc)(struct _typeobject *, int);
 
+getbufferproc = Signature("TBi", "r")      # typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
+releasebufferproc = Signature("TB", "v")   # typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+
+
 #------------------------------------------------------------------------------------------
 #
 #  Signatures for accessor methods of properties.
@@ -515,7 +539,7 @@ PyNumberMethods = (
     MethodSlot(binaryfunc, "nb_add", "__add__"),
     MethodSlot(binaryfunc, "nb_subtract", "__sub__"),
     MethodSlot(binaryfunc, "nb_multiply", "__mul__"),
-    MethodSlot(binaryfunc, "nb_divide", "__div__"),
+    MethodSlot(binaryfunc, "nb_divide", "__div__", py3k = False),
     MethodSlot(binaryfunc, "nb_remainder", "__mod__"),
     MethodSlot(binaryfunc, "nb_divmod", "__divmod__"),
     MethodSlot(ternaryfunc, "nb_power", "__pow__"),
@@ -540,7 +564,7 @@ PyNumberMethods = (
     MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"),
     MethodSlot(ibinaryfunc, "nb_inplace_subtract", "__isub__"),
     MethodSlot(ibinaryfunc, "nb_inplace_multiply", "__imul__"),
-    MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__"),
+    MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", py3k = False),
     MethodSlot(ibinaryfunc, "nb_inplace_remainder", "__imod__"),
     MethodSlot(ternaryfunc, "nb_inplace_power", "__ipow__"), # NOT iternaryfunc!!!
     MethodSlot(ibinaryfunc, "nb_inplace_lshift", "__ilshift__"),
@@ -580,10 +604,13 @@ PyMappingMethods = (
 )
 
 PyBufferProcs = (
-    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__"),
-    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__"),
-    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__"),
-    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__"),
+    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3k = False),
+    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3k = False),
+    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__", py3k = False),
+    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3k = False),
+
+    MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", flag = "Py_TPFLAGS_HAVE_NEWBUFFER"),
+    MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", flag = "Py_TPFLAGS_HAVE_NEWBUFFER"),
 )
 
 #------------------------------------------------------------------------------------------

Cython/Utils.py

@@ -91,3 +91,20 @@ class EncodedString(unicode):
 #    def __eq__(self, other):
 #        return unicode.__eq__(self, other) and \
 #            getattr(other, 'encoding', '') == self.encoding
+
+def escape_byte_string(s):
+    s = s.replace('\0', r'\x00')
+    try:
+        s.decode("ASCII")
+        return s
+    except UnicodeDecodeError:
+        pass
+    l = []
+    append = l.append
+    for c in s:
+        o = ord(c)
+        if o >= 128:
+            append('\\x%X' % o)
+        else:
+            append(c)
+    return ''.join(l)

Makefile

@@ -17,3 +17,7 @@ testclean:
 
 test:	testclean
 	${PYTHON} runtests.py
+
+test3:	testclean
+	${PYTHON} runtests.py --no-cleanup
+	python3.0 runtests.py --no-cython

runtests.py

@@ -2,11 +2,7 @@
 
 import os, sys, re, shutil, unittest, doctest
 
-from Cython.Compiler.Version import version
-from Cython.Compiler.Main import \
-    CompilationOptions, \
-    default_options as pyrex_default_options, \
-    compile as cython_compile
+WITH_CYTHON = True
 
 from distutils.dist import Distribution
 from distutils.core import Extension
@@ -37,17 +33,21 @@ class ErrorWriter(object):
         return errors
 
 class TestBuilder(object):
-    def __init__(self, rootdir, workdir, selectors, annotate):
+    def __init__(self, rootdir, workdir, selectors, annotate, cleanup_workdir):
         self.rootdir = rootdir
         self.workdir = workdir
         self.selectors = selectors
         self.annotate = annotate
+        self.cleanup_workdir = cleanup_workdir
 
     def build_suite(self):
         suite = unittest.TestSuite()
         filenames = os.listdir(self.rootdir)
         filenames.sort()
         for filename in filenames:
+            if not WITH_CYTHON and filename == "errors":
+                # we won't get any errors without running Cython
+                continue
             path = os.path.join(self.rootdir, filename)
             if os.path.isdir(path) and filename in TEST_DIRS:
                 suite.addTest(
@@ -55,6 +55,12 @@ class TestBuilder(object):
         return suite
 
     def handle_directory(self, path, context):
+        workdir = os.path.join(self.workdir, context)
+        if not os.path.exists(workdir):
+            os.makedirs(workdir)
+        if workdir not in sys.path:
+            sys.path.insert(0, workdir)
+
         expect_errors = (context == 'errors')
         suite = unittest.TestSuite()
         filenames = os.listdir(path)
@@ -69,29 +75,38 @@ class TestBuilder(object):
                 continue
             if context in TEST_RUN_DIRS:
                 test = CythonRunTestCase(
-                    path, self.workdir, module, self.annotate)
+                    path, workdir, module,
+                    annotate=self.annotate,
+                    cleanup_workdir=self.cleanup_workdir)
             else:
                 test = CythonCompileTestCase(
-                    path, self.workdir, module, expect_errors, self.annotate)
+                    path, workdir, module,
+                    expect_errors=expect_errors,
+                    annotate=self.annotate,
+                    cleanup_workdir=self.cleanup_workdir)
             suite.addTest(test)
         return suite
 
 class CythonCompileTestCase(unittest.TestCase):
     def __init__(self, directory, workdir, module,
-                 expect_errors=False, annotate=False):
+                 expect_errors=False, annotate=False, cleanup_workdir=True):
         self.directory = directory
         self.workdir = workdir
         self.module = module
         self.expect_errors = expect_errors
         self.annotate = annotate
+        self.cleanup_workdir = cleanup_workdir
         unittest.TestCase.__init__(self)
 
     def shortDescription(self):
         return "compiling " + self.module
 
     def tearDown(self):
+        cleanup_c_files = WITH_CYTHON and self.cleanup_workdir
         if os.path.exists(self.workdir):
             for rmfile in os.listdir(self.workdir):
+                if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
+                    continue
                 if self.annotate and rmfile.endswith(".html"):
                     continue
                 try:
@@ -171,6 +186,7 @@ class CythonCompileTestCase(unittest.TestCase):
                 directory, module, workdir)
             directory = workdir
 
+        if WITH_CYTHON:
             old_stderr = sys.stderr
             try:
                 sys.stderr = ErrorWriter()
@@ -198,11 +214,11 @@ class CythonRunTestCase(CythonCompileTestCase):
     def run(self, result=None):
         if result is None:
             result = self.defaultTestResult()
+        result.startTest(self)
         try:
             self.runTest()
             doctest.DocTestSuite(self.module).run(result)
         except Exception:
-            result.startTest(self)
             result.addError(self, sys.exc_info())
             result.stopTest(self)
         try:
@@ -211,49 +227,68 @@ class CythonRunTestCase(CythonCompileTestCase):
             pass
 
 if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser()
+    parser.add_option("--no-cleanup", dest="cleanup_workdir",
+                      action="store_false", default=True,
+                      help="do not delete the generated C files (allows passing --no-cython on next run)")
+    parser.add_option("--no-cython", dest="with_cython",
+                      action="store_false", default=True,
+                      help="do not run the Cython compiler, only the C compiler")
+    parser.add_option("-C", "--coverage", dest="coverage",
+                      action="store_true", default=False,
+                      help="collect source coverage data for the Compiler")
+    parser.add_option("-A", "--annotate", dest="annotate_source",
+                      action="store_true", default=False,
+                      help="generate annotated HTML versions of the test source files")
+    parser.add_option("-v", "--verbose", dest="verbosity",
+                      action="count", default=0,
+                      help="display test progress, pass twice to print test names")
+
+    options, cmd_args = parser.parse_args()
+
+    if options.coverage:
+        import coverage
+        coverage.erase()
+        coverage.start()
+
+    WITH_CYTHON = options.with_cython
+
+    if WITH_CYTHON:
+        from Cython.Compiler.Main import \
+            CompilationOptions, \
+            default_options as pyrex_default_options, \
+            compile as cython_compile
+
     # RUN ALL TESTS!
     ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
     WORKDIR = os.path.join(os.getcwd(), 'BUILD')
+    if WITH_CYTHON:
         if os.path.exists(WORKDIR):
             shutil.rmtree(WORKDIR, ignore_errors=True)
+    if not os.path.exists(WORKDIR):
         os.makedirs(WORKDIR)
 
-    if not sys.path or sys.path[0] != WORKDIR:
-        sys.path.insert(0, WORKDIR)
-
-    print "Running tests against Cython %s" % version
-    print "Python", sys.version
-    print
-
-    try:
-        sys.argv.remove("-C")
-    except ValueError:
-        coverage = None
+    if WITH_CYTHON:
+        from Cython.Compiler.Version import version
+        print("Running tests against Cython %s" % version)
     else:
-        import coverage
-        coverage.erase()
-
-    try:
-        sys.argv.remove("-a")
-    except ValueError:
-        annotate_source = False
-    else:
-        annotate_source = True
+        print("Running tests without Cython.")
+    print("Python %s" % sys.version)
+    print("")
 
     import re
-    selectors = [ re.compile(r, re.I).search for r in sys.argv[1:] ]
+    selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
     if not selectors:
         selectors = [ lambda x:True ]
 
-    tests = TestBuilder(ROOTDIR, WORKDIR, selectors, annotate_source)
+    tests = TestBuilder(ROOTDIR, WORKDIR, selectors,
+                        options.annotate_source, options.cleanup_workdir)
     test_suite = tests.build_suite()
 
-    if coverage is not None:
-        coverage.start()
-
-    unittest.TextTestRunner(verbosity=2).run(test_suite)
+    unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 
-    if coverage is not None:
+    if options.coverage:
         coverage.stop()
         ignored_modules = ('Options', 'Version', 'DebugFlags')
         modules = [ module for name, module in sys.modules.items()

tests/broken/ass2cglobal.pyx

-cdef int i
-cdef x
-
-def f(a):
-    global i, x
-    i = 42
-    x = a

tests/broken/cstringmeth.pyx

-x = "foo".join(y)
-

tests/broken/filenames.pxi

-spam = "ftang"

tests/broken/inhcmethcall.pyx

-cdef class Parrot:
-
-  cdef void describe(self):
-    pass
-
-
-cdef class Norwegian(Parrot):
-
-  cdef void describe(self):
-    Parrot.describe(self)

tests/compile/classmethargdefault.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Swallow()
     >>> s.spam(1)
     1 42 'grail' True

tests/compile/dotted_cimport.pyx

+cimport dotted_cimport_submodule.a
+import dotted_cimport_submodule.b

tests/broken/eqcmp.pyx → tests/compile/eqcmp.pyx

@@ -4,7 +4,6 @@ cdef void foo():
     cdef char *ptr1, *ptr2
     cdef int *ptr3
     bool = int1 == int2
-    bool = int1 <> int2
     bool = int1 != int2
     bool = float1 == float2
     bool = ptr1 == ptr2

tests/broken/extcoerce.pyx → tests/compile/extcoerce.pyx

@@ -3,12 +3,6 @@ cdef class Grail:
     def __add__(int x, float y):
         pass
 
-    def __getslice__(self, i, j):
-        pass
-
-    def __setslice__(self, Py_ssize_t i, float j, x):
-        pass
-
 cdef class Swallow:
     pass
     

tests/compile/extinheritdel.pyx

@@ -6,9 +6,6 @@ cdef class Norwegian(Parrot):
     def __delitem__(self, i):
         pass
 
-    def __delslice__(self, i, j):
-        pass
-
     def __delattr__(self, n):
         pass
 

tests/compile/extinheritset.pyx

@@ -6,9 +6,6 @@ cdef class Norwegian(Parrot):
     def __setitem__(self, i, x):
         pass
 
-    def __setslice__(self, i, j, x):
-        pass
-
     def __setattr__(self, n, x):
         pass
 

tests/compile/food.h

+struct Tomato {
	PyObject_HEAD};
struct Bicycle{
	PyObject_HEAD};
\ No newline at end of file

tests/compile/pinard4.pyx

-__doc__ = """
+__doc__ = u"""
     >>> fiches_CP
     []
 """

tests/broken/r_pernici1.pyx → tests/compile/r_pernici1.pyx

+__doc__ = u"""
+>>> main()
+3.14159265358979323846
+3.14159265358979323846
+3.14159265358979323846
+"""
+
 cdef extern from "math.h":
   double M_PI
 

tests/errors/cdefoptargs.pyx

+def call5():
+    b(1,2,3,4,5)
+
+cdef b(a, b, c=1, d=2):
+    pass
+
+_ERRORS = """
+2:5:Call with wrong number of arguments (expected at most 4, got 5)
+"""

tests/run/__getattribute__.pyx

-__doc__ = """
+__doc__ = u"""
 __getattribute__ and __getattr__ special methods for a single class.
 
     >>> a = just_getattribute()
@@ -29,7 +29,7 @@ __getattribute__ and __getattr__ special methods for a single class.
 
 cdef class just_getattribute:
     def __getattribute__(self,n):
-        if n == 'bar':
+        if n == u'bar':
             return n
         else:
             raise AttributeError
@@ -39,7 +39,7 @@ cdef class just_getattr:
     def __init__(self):
         self.foo = 10
     def __getattr__(self,n):
-        if n == 'bar':
+        if n == u'bar':
             return n
         else:
             raise AttributeError
@@ -49,12 +49,12 @@ cdef class both:
     def __init__(self):
         self.foo = 10
     def __getattribute__(self,n):
-        if n == 'foo':
+        if n == u'foo':
             return self.foo
         else:
             raise AttributeError
     def __getattr__(self,n):
-        if n == 'bar':
+        if n == u'bar':
             return n
         else:
             raise AttributeError

tests/run/__getattribute_subclasses__.pyx

-__doc__ = """
+__doc__ = u"""
 __getattribute__ and __getattr__ special methods and subclasses. 
 
 getattr does not override members. 
     >>> a = getattr_boring()
     >>> a.boring_member
     10
-    >>> a.resolved_by
-    'getattr_boring'
+    >>> print(a.resolved_by)
+    getattr_boring
 
 getattribute does.
     >>> a = getattribute_boring()
     >>> a.boring_member
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    'getattribute_boring'
+    >>> print(a.resolved_by)
+    getattribute_boring
 
 Is inherited.
     >>> a = boring_boring_getattribute()
@@ -24,8 +24,8 @@ Is inherited.
     >>> a.boring_boring_getattribute_member
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    '_getattribute'
+    >>> print(a.resolved_by)
+    _getattribute
 
 __getattribute__ is always tried first, then __getattr__, regardless of where
 in the inheritance hiarchy they came from.
@@ -33,8 +33,8 @@ in the inheritance hiarchy they came from.
     >>> a.foo
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    'getattribute_boring_boring_getattr'
+    >>> print(a.resolved_by)
+    getattribute_boring_boring_getattr
     >>> a.getattribute_boring_boring_getattr
     True
     >>> a._getattr
@@ -44,8 +44,8 @@ in the inheritance hiarchy they came from.
     >>> a.foo
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    '_getattribute'
+    >>> print(a.resolved_by)
+    _getattribute
     >>> a.getattr_boring_boring_getattribute
     True
     >>> a._getattribute
@@ -60,36 +60,36 @@ cdef class boring:
 
 cdef class getattr_boring(boring):
     def __getattr__(self,n):
-        if n == 'resolved_by':
-            return 'getattr_boring'
-        elif n == 'getattr_boring':
+        if n == u'resolved_by':
+            return u'getattr_boring'
+        elif n == u'getattr_boring':
             return True
         else:
             raise AttributeError
 
 cdef class getattribute_boring(boring):
     def __getattribute__(self,n):
-        if n == 'resolved_by':
-            return 'getattribute_boring'
-        elif n == 'getattribute_boring':
+        if n == u'resolved_by':
+            return u'getattribute_boring'
+        elif n == u'getattribute_boring':
             return True
         else:
             raise AttributeError
 
 cdef class _getattr:
     def __getattr__(self,n):
-        if n == 'resolved_by':
-            return '_getattr'
-        elif n == '_getattr':
+        if n == u'resolved_by':
+            return u'_getattr'
+        elif n == u'_getattr':
             return True
         else:
             raise AttributeError
 
 cdef class _getattribute(boring):
     def __getattribute__(self,n):
-        if n == 'resolved_by':
-            return '_getattribute'
-        elif n == '_getattribute':
+        if n == u'resolved_by':
+            return u'_getattribute'
+        elif n == u'_getattribute':
             return True
         else:
             raise AttributeError
@@ -108,19 +108,18 @@ cdef class boring_boring_getattr(boring_getattr):
 
 cdef class getattribute_boring_boring_getattr(boring_boring_getattr):
     def __getattribute__(self,n):
-        if n == 'resolved_by':
-            return 'getattribute_boring_boring_getattr'
-        elif n == 'getattribute_boring_boring_getattr':
+        if n == u'resolved_by':
+            return u'getattribute_boring_boring_getattr'
+        elif n == u'getattribute_boring_boring_getattr':
             return True
         else:
             raise AttributeError
 
 cdef class getattr_boring_boring_getattribute(boring_boring_getattribute):
     def __getattr__(self,n):
-        if n == 'resolved_by':
-            return 'getattr_boring_boring_getattribute'
-        elif n == 'getattr_boring_boring_getattribute':
+        if n == u'resolved_by':
+            return u'getattr_boring_boring_getattribute'
+        elif n == u'getattr_boring_boring_getattribute':
             return True
         else:
             raise AttributeError
-

tests/run/addloop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> x = 1
     >>> for i in range(10):
     ...     x = x + i

tests/run/addop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     (30, 22)
 """

tests/run/addressof.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(5)
     5
 """

tests/run/altet2.pyx

-__doc__ = """
+__doc__ = u"""
   >>> iter(C())
   Traceback (most recent call last):
   TypeError: iter() returned non-iterator of type 'NoneType'

tests/broken/anonymousenum.pyx → tests/run/anonymousenum.pyx

+__doc__ = u"""
+>>> p
+42
+"""
+
 cdef enum:
     spam = 42
     grail = 17
@@ -5,3 +10,4 @@ cdef enum:
 cdef int i
 i = spam
 
+p = i

tests/run/append.pyx

-__doc__ = """
+__doc__ = u"""
 >>> test_append([])
 None
 None
@@ -23,7 +23,7 @@ None
 
 class A:
     def append(self, x):
-        print "appending"
+        print u"appending"
         return x
 
 class B(list):
@@ -38,6 +38,6 @@ def test_append(L):
     try:
         print L.append(5,6)
     except TypeError:
-        print "got error"
+        print u"got error"
     return L
 

tests/run/ass2cglobal.pyx

+__doc__ = u"""
+>>> what()
+0 5
+>>> f(5)
+>>> what()
+42 5
+>>> f(6)
+>>> what()
+42 6
+>>> f("spam")
+>>> what()
+42 spam
+"""
+
+cdef int i = 0
+cdef x = 5
+
+def f(a):
+    global i, x
+    i = 42
+    x = a
+
+def what():
+    print i,x

tests/run/ass2global.pyx

-__doc__ = """
+__doc__ = u"""
     >>> getg()
     5
     >>> f(42)

tests/run/ass2local.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     42
 """

tests/run/assert.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1, 2, 1)
     >>> f(0, 2, 1)
     Traceback (most recent call last):

tests/run/attr.pyx

-__doc__ = """
-    >>> class Test:
+__doc__ = u"""
+    >>> class Test(object):
     ...     def __init__(self, i):
     ...         self.i = i
     >>> b = Test(1)
@@ -9,50 +9,50 @@ __doc__ = """
     >>> b.spam.eggs.spam.eggs = Test(5)
 
     >>> a = f(b)
-    >>> print a.i
+    >>> a.i
     2
-    >>> print b.i
+    >>> b.i
     1
-    >>> print a.spam.i
+    >>> a.spam.i
     1
-    >>> print b.spam.i
+    >>> b.spam.i
     2
-    >>> print a.spam.eggs.i
+    >>> a.spam.eggs.i
     Traceback (most recent call last):
-    AttributeError: Test instance has no attribute 'eggs'
-    >>> print b.spam.eggs.i
+    AttributeError: 'Test' object has no attribute 'eggs'
+    >>> b.spam.eggs.i
     3
-    >>> print a.spam.spam.i
+    >>> a.spam.spam.i
     2
-    >>> print b.spam.spam.i
+    >>> b.spam.spam.i
     1
-    >>> print a.spam.eggs.spam.i
+    >>> a.spam.eggs.spam.i
     Traceback (most recent call last):
-    AttributeError: Test instance has no attribute 'eggs'
-    >>> print b.spam.eggs.spam.i
+    AttributeError: 'Test' object has no attribute 'eggs'
+    >>> b.spam.eggs.spam.i
     4
 
     >>> a = g(b)
-    >>> print a.i
+    >>> a.i
     3
-    >>> print b.i
+    >>> b.i
     1
-    >>> print a.spam.i
+    >>> a.spam.i
     4
-    >>> print b.spam.i
+    >>> b.spam.i
     2
-    >>> print a.spam.eggs.i
+    >>> a.spam.eggs.i
     1
-    >>> print b.spam.eggs.i
+    >>> b.spam.eggs.i
     3
-    >>> print a.spam.spam.i
+    >>> a.spam.spam.i
     Traceback (most recent call last):
-    AttributeError: Test instance has no attribute 'spam'
-    >>> print b.spam.spam.i
+    AttributeError: 'Test' object has no attribute 'spam'
+    >>> b.spam.spam.i
     1
-    >>> print a.spam.eggs.spam.i
+    >>> a.spam.eggs.spam.i
     2
-    >>> print b.spam.eggs.spam.i
+    >>> b.spam.eggs.spam.i
     4
 """
 

tests/run/baas3.pyx

-__doc__ = """
+__doc__ = u"""
     >>> m = MyClass()
     >>> m is foo(m)
     True

tests/run/backquote.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(20)
     '20'
     >>> f('test')

tests/run/behnel1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> viking(5)
     5
 """

tests/run/behnel2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> y
     1
     >>> y and {}

tests/run/behnel3.pyx

-__doc__ = """
+__doc__ = u"""
     >>> y
     >>> y or {}
     {}

tests/run/bishop1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> m = fmatrix()
     >>> m[1] = True
     >>> m.getfoo()

tests/run/bishop2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f = foo()
     >>> 'a' in f
     True

tests/run/boolop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> foo(True, False, 23, 'test', 1)
     (0, 1, False, False)
 """

tests/run/buffer.pyx

+__doc__ = u"""
+>>> b1 = TestBuffer()
+>>> b2 = TestBufferRelease()
+"""
+
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ += u"""
+>>> ms = memoryview(s)
+>>> ms.tobytes()
+bytearray(b'abcdefg')
+
+>>> m1 = memoryview(b1)
+>>> m1.tobytes()
+locking!
+bytearray(b'abcdefg')
+
+>>> m2 = memoryview(b2)
+>>> m2.tobytes()
+locking!
+unlocking!
+bytearray(b'abcdefg')
+
+>>> del m1
+>>> del m2
+releasing!
+"""
+
+s = "abcdefg"
+
+cdef class TestBuffer:
+    def __getbuffer__(self, Py_buffer* buffer, int flags):
+        if buffer is NULL:
+            print u"locking!"
+            return
+        buffer.buf = <char*>s
+        buffer.len = len(s)
+        buffer.readonly = 0
+        buffer.format = "B"
+        buffer.ndim = 0
+        buffer.shape = NULL
+        buffer.strides = NULL
+        buffer.suboffsets = NULL
+        buffer.itemsize = 1
+        buffer.internal = NULL
+
+cdef class TestBufferRelease(TestBuffer):
+    def __releasebuffer__(self, Py_buffer* buffer):
+        if buffer is NULL:
+            print u"unlocking!"
+        else:
+            print u"releasing!"

tests/run/cdefoptargs.pyx

+__doc__ = u"""
+    >>> call2()
+    >>> call3()
+    >>> call4()
+"""
+
+# the calls:
+
+def call2():
+    b(1,2)
+
+def call3():
+    b(1,2,3)
+
+def call4():
+    b(1,2,3,4)
+
+# the called function:
+
+cdef b(a, b, c=1, d=2):
+    pass

tests/run/cfuncdef.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test()
 """
 

tests/run/cintop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> int2 = 42
     >>> int3 = 7
     >>> char1 = ord('C')
@@ -10,7 +10,7 @@ __doc__ = """
     >>> int1 ^= int2 >> int3
     >>> int1 ^= int2 << int3 | int2 >> int3
     >>> long1 = char1 | int1
-    >>> print (int1, long1) == f()
+    >>> (int1, long1) == f()
     True
 
     >>> f()

tests/run/classkwonlyargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> spam = Spam()
     >>> b,c,d,e,f,g,h,k = spam.b,spam.c,spam.d,spam.e,spam.f,spam.g,spam.h,spam.k
 
@@ -49,7 +49,7 @@ __doc__ = """
     >>> g(1,2, c=1, e=0, f=2, d=11)
     >>> g(1,2, c=1, f=2, e=0, x=25)
 
-    >>> g(1,2,3)
+    >>> g(1,2,3) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes at most 3 positional arguments (4 given)
     >>> g(1,2)
@@ -84,6 +84,14 @@ __doc__ = """
     TypeError: required keyword argument 'f' is missing
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__, re.M)
+
 class Spam:
     def b(self, a, b, c):
         pass

tests/run/classmethod.pyx

+__doc__ = u"""
+>>> class1.view()
+class1
+>>> class1.plus(1)
+6
+>>> class2.view()
+class2
+>>> class2.plus(1)
+7
+>>> class3.view()
+class3
+>>> class3.plus(1)
+8
+"""
+
+def f_plus(cls, a):
+    return cls.a + a
+
+class class1:
+    a = 5
+    plus = classmethod(f_plus)
+    def view(cls):
+        print cls.__name__
+    view = classmethod(view)
+
+class class2(object):
+    a = 6
+    plus = classmethod(f_plus)
+    def view(cls):
+        print cls.__name__
+    view = classmethod(view)
+
+cdef class class3:
+    a = 7
+    plus = classmethod(f_plus)
+    def view(cls):
+        print cls.__name__
+    view = classmethod(view)

tests/run/classpass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Spam()
-    >>> print s.__class__.__name__
-    Spam
+    >>> s.__class__.__name__
+    'Spam'
 
     >>> s = SpamT()
-    >>> print type(s).__name__
-    SpamT
+    >>> type(s).__name__
+    'SpamT'
 """
 
 class Spam: pass

tests/run/compiledef.pyx

-__doc__ = """
+__doc__ = u"""
     >>> t
     True
     >>> f