Merge

Cython/CodeWriter.py

@@ -149,8 +149,8 @@ class CodeWriter(TreeVisitor):
     
     def visit_PrintStatNode(self, node):
         self.startline(u"print ")
-        self.comma_seperated_list(node.args)
-        if node.ends_with_comma:
+        self.comma_seperated_list(node.arg_tuple.args)
+        if not node.append_newline:
             self.put(u",")
         self.endline()
 

Cython/Compiler/Builtin.py

@@ -2,8 +2,9 @@
 #   Pyrex - Builtin Definitions
 #
 
-from Symtab import BuiltinScope
+from Symtab import BuiltinScope, StructOrUnionScope
 from TypeSlots import Signature
+import PyrexTypes
 
 builtin_function_table = [
     # name,        args,   return,  C API func,           py equiv = "*"
@@ -24,7 +25,7 @@ builtin_function_table = [
     #('hex',       "",     "",      ""),
     #('id',        "",     "",      ""),
     #('input',     "",     "",      ""),
-    ('intern',     "s",    "O",     "PyString_InternFromString"),
+    ('intern',     "s",    "O",     "__Pyx_InternFromString"),
     ('isinstance', "OO",   "b",     "PyObject_IsInstance"),
     ('issubclass', "OO",   "b",     "PyObject_IsSubclass"),
     ('iter',       "O",    "O",     "PyObject_GetIter"),
@@ -100,6 +101,21 @@ builtin_types_table = [
                                     ("values","O",   "O", "PyDict_Values")]),
 ]
 
+builtin_structs_table = [
+    ('Py_buffer', 'Py_buffer',
+     [("buf",        PyrexTypes.c_void_ptr_type),
+      ("len",        PyrexTypes.c_py_ssize_t_type),
+      ("readonly",   PyrexTypes.c_bint_type),
+      ("format",     PyrexTypes.c_char_ptr_type),
+      ("ndim",       PyrexTypes.c_int_type),
+      ("shape",      PyrexTypes.c_py_ssize_t_ptr_type),
+      ("strides",    PyrexTypes.c_py_ssize_t_ptr_type),
+      ("suboffsets", PyrexTypes.c_py_ssize_t_ptr_type),
+      ("itemsize",   PyrexTypes.c_py_ssize_t_type),
+      ("internal",   PyrexTypes.c_void_ptr_type),
+      ])
+]
+
 getattr3_utility_code = ["""
 static PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/
 ""","""
@@ -118,8 +134,18 @@ bad:
 }
 """]
 
+intern_utility_code = ["""
+#if PY_MAJOR_VERSION >= 3
+#  define __Pyx_InternFromString(s) PyUnicode_InternFromString(s)
+#else
+#  define __Pyx_InternFromString(s) PyString_InternFromString(s)
+#endif
+""","""
+"""]
+
 builtin_utility_code = {
     'getattr3': getattr3_utility_code,
+    'intern'  : intern_utility_code,
 }
 
 builtin_scope = BuiltinScope()
@@ -141,12 +167,22 @@ def init_builtin_types():
             sig = Signature(args, ret)
             the_type.scope.declare_cfunction(name, sig.function_type(), None, cname)
 
+def init_builtin_structs():
+    for name, cname, attribute_types in builtin_structs_table:
+        scope = StructOrUnionScope(name)
+        for attribute_name, attribute_type in attribute_types:
+            scope.declare_var(
+                attribute_name, attribute_type, None, attribute_name)
+        builtin_scope.declare_struct_or_union(
+            name, "struct", scope, 1, None, cname = cname)
+
 def init_builtins():
     init_builtin_funcs()
     init_builtin_types()
+    init_builtin_structs()
     global list_type, tuple_type, dict_type
     list_type  = builtin_scope.lookup('list').type
     tuple_type = builtin_scope.lookup('tuple').type
     dict_type  = builtin_scope.lookup('dict').type
-    
+
 init_builtins()

Cython/Compiler/ExprNodes.py

@@ -739,6 +739,18 @@ class StringNode(ConstNode):
             return self.entry.cname
 
 
+class IdentifierStringNode(ConstNode):
+    # A Python string that behaves like an identifier, e.g. for
+    # keyword arguments in a call, or for imported names
+    type = PyrexTypes.py_object_type
+
+    def analyse_types(self, env):
+        self.cname = env.intern_identifier(self.value)
+
+    def calculate_result_code(self):
+        return self.cname
+
+
 class LongNode(AtomicExprNode):
     #  Python long integer literal
     #
@@ -838,6 +850,8 @@ class NameNode(AtomicExprNode):
         env.control_flow.set_state(self.pos, (self.name, 'source'), 'assignment')
         if self.entry.is_declared_generic:
             self.result_ctype = py_object_type
+        if self.entry.is_pyglobal and self.entry.is_member:
+            env.use_utility_code(type_cache_invalidation_code)
     
     def analyse_types(self, env):
         self.entry = env.lookup(self.name)
@@ -868,10 +882,7 @@ class NameNode(AtomicExprNode):
                 self.is_temp = 0
             else:
                 self.is_temp = 1
-            if Options.intern_names:
-                env.use_utility_code(get_name_interned_utility_code)
-            else:
-                env.use_utility_code(get_name_utility_code)
+            env.use_utility_code(get_name_interned_utility_code)
     
     def analyse_entry(self, env):
         #print "NameNode.analyse_entry:", self.name ###
@@ -881,8 +892,8 @@ class NameNode(AtomicExprNode):
         self.type = type
         if entry.is_pyglobal or entry.is_builtin:
             assert type.is_pyobject, "Python global or builtin not a Python object"
-            if Options.intern_names:
-                self.interned_cname = self.entry.interned_cname = env.intern(self.entry.name)
+            self.interned_cname = self.entry.interned_cname = \
+                env.intern_identifier(self.entry.name)
 
     def check_identifier_kind(self):
         #print "NameNode.check_identifier_kind:", self.entry.name ###
@@ -950,20 +961,12 @@ class NameNode(AtomicExprNode):
                 namespace = Naming.builtins_cname
             else: # entry.is_pyglobal
                 namespace = entry.namespace_cname
-            if Options.intern_names:
-                code.putln(
-                    '%s = __Pyx_GetName(%s, %s); %s' % (
-                    self.result_code,
-                    namespace, 
-                    self.interned_cname,
-                    code.error_goto_if_null(self.result_code, self.pos)))		
-            else:
-                code.putln(
-                    '%s = __Pyx_GetName(%s, "%s"); %s' % (
-                    self.result_code,
-                    namespace, 
-                    self.entry.name,
-                    code.error_goto_if_null(self.result_code, self.pos)))
+            code.putln(
+                '%s = __Pyx_GetName(%s, %s); %s' % (
+                self.result_code,
+                namespace, 
+                self.interned_cname,
+                code.error_goto_if_null(self.result_code, self.pos)))		
         elif entry.is_local and False:
             # control flow not good enough yet
             assigned = entry.scope.control_flow.get_state((entry.name, 'initalized'), self.pos)
@@ -984,34 +987,22 @@ class NameNode(AtomicExprNode):
         if entry.is_pyglobal:
             namespace = self.entry.namespace_cname
             if entry.is_member:
-                # if we entry is a member we have to cheat: SetAttr does not work
+                # if the entry is a member we have to cheat: SetAttr does not work
                 # on types, so we create a descriptor which is then added to tp_dict
-                if Options.intern_names:
-                    code.put_error_if_neg(self.pos,
-                        'PyDict_SetItem(%s->tp_dict, %s, %s)' % (
-                            namespace,
-                            self.interned_cname,
-                            rhs.py_result()))
-                else:
-                    code.put_error_if_neg(self.pos,
-                        'PyDict_SetItemString(%s->tp_dict, %s, %s)' % (
-                            namespace,
-                            entry.name,
-                            rhs.py_result()))
-
+                code.put_error_if_neg(self.pos,
+                    'PyDict_SetItem(%s->tp_dict, %s, %s)' % (
+                        namespace,
+                        self.interned_cname,
+                        rhs.py_result()))
+                # in Py2.6+, we need to invalidate the method cache
+                code.putln("__Pyx_TypeModified((PyTypeObject*)%s);" %
+                           entry.scope.parent_type.typeptr_cname)
             else: 
-                if Options.intern_names:
-                    code.put_error_if_neg(self.pos,
-                        'PyObject_SetAttr(%s, %s, %s)' % (
-                            namespace,
-                            self.interned_cname,
-                            rhs.py_result()))
-                else:
-                    code.put_error_if_neg(self.pos,
-                        'PyObject_SetAttrString(%s, "%s", %s)' % (
-                            namespace, 
-                            entry.name,
-                            rhs.py_result()))
+                code.put_error_if_neg(self.pos,
+                    'PyObject_SetAttr(%s, %s, %s)' % (
+                        namespace,
+                        self.interned_cname,
+                        rhs.py_result()))
                 if debug_disposal_code:
                     print("NameNode.generate_assignment_code:")
                     print("...generating disposal code for %s" % rhs)
@@ -1084,8 +1075,8 @@ class ImportNode(ExprNode):
     #  Implements result = 
     #    __import__(module_name, globals(), None, name_list)
     #
-    #  module_name   StringNode         dotted name of module
-    #  name_list     ListNode or None   list of names to be imported
+    #  module_name   IdentifierStringNode     dotted name of module
+    #  name_list     ListNode or None         list of names to be imported
     
     subexprs = ['module_name', 'name_list']
 
@@ -1836,7 +1827,7 @@ class AttributeNode(ExprNode):
     #  member               string    C name of struct member
     #  is_called            boolean   Function call is being done on result
     #  entry                Entry     Symbol table entry of attribute
-    #  interned_attr_cname	string    C name of interned attribute name
+    #  interned_attr_cname  string    C name of interned attribute name
     
     is_attribute = 1
     subexprs = ['obj']
@@ -2019,8 +2010,7 @@ class AttributeNode(ExprNode):
         if obj_type.is_pyobject:
             self.type = py_object_type
             self.is_py_attr = 1
-            if Options.intern_names:
-                self.interned_attr_cname = env.intern(self.attribute)
+            self.interned_attr_cname = env.intern_identifier(self.attribute)
         else:
             if not obj_type.is_error:
                 error(self.pos, 
@@ -2064,36 +2054,21 @@ class AttributeNode(ExprNode):
     
     def generate_result_code(self, code):
         if self.is_py_attr:
-            if Options.intern_names:
-                code.putln(
-                    '%s = PyObject_GetAttr(%s, %s); %s' % (
-                        self.result_code,
-                        self.obj.py_result(),
-                        self.interned_attr_cname,
-                        code.error_goto_if_null(self.result_code, self.pos)))
-            else:
-                code.putln(
-                    '%s = PyObject_GetAttrString(%s, "%s"); %s' % (
-                        self.result_code,
-                        self.obj.py_result(),
-                        self.attribute,
-                        code.error_goto_if_null(self.result_code, self.pos)))
+            code.putln(
+                '%s = PyObject_GetAttr(%s, %s); %s' % (
+                    self.result_code,
+                    self.obj.py_result(),
+                    self.interned_attr_cname,
+                    code.error_goto_if_null(self.result_code, self.pos)))
     
     def generate_assignment_code(self, rhs, code):
         self.obj.generate_evaluation_code(code)
         if self.is_py_attr:
-            if Options.intern_names:
-                code.put_error_if_neg(self.pos, 
-                    'PyObject_SetAttr(%s, %s, %s)' % (
-                        self.obj.py_result(),
-                        self.interned_attr_cname,
-                        rhs.py_result()))
-            else:
-                code.put_error_if_neg(self.pos,
-                    'PyObject_SetAttrString(%s, "%s", %s)' % (
-                        self.obj.py_result(),
-                        self.attribute,
-                        rhs.py_result()))
+            code.put_error_if_neg(self.pos, 
+                'PyObject_SetAttr(%s, %s, %s)' % (
+                    self.obj.py_result(),
+                    self.interned_attr_cname,
+                    rhs.py_result()))
             rhs.generate_disposal_code(code)
         else:
             select_code = self.result_code
@@ -2111,16 +2086,10 @@ class AttributeNode(ExprNode):
     def generate_deletion_code(self, code):
         self.obj.generate_evaluation_code(code)
         if self.is_py_attr:
-            if Options.intern_names:
-                code.put_error_if_neg(self.pos,
-                    'PyObject_DelAttr(%s, %s)' % (
-                        self.obj.py_result(),
-                        self.interned_attr_cname))
-            else:
-                code.put_error_if_neg(self.pos, 
-                    'PyObject_DelAttrString(%s, "%s")' % (
-                        self.obj.py_result(),
-                        self.attribute))
+            code.put_error_if_neg(self.pos,
+                'PyObject_DelAttr(%s, %s)' % (
+                    self.obj.py_result(),
+                    self.interned_attr_cname))
         else:
             error(self.pos, "Cannot delete C attribute of extension type")
         self.obj.generate_disposal_code(code)
@@ -2463,22 +2432,23 @@ class DictItemNode(ExprNode):
         self.key.generate_disposal_code(code)
         self.value.generate_disposal_code(code)
 
+
 class ClassNode(ExprNode):
     #  Helper class used in the implementation of Python
     #  class definitions. Constructs a class object given
     #  a name, tuple of bases and class dictionary.
     #
-    #  name         ExprNode           Name of the class
+    #  name         EncodedString      Name of the class
+    #  cname        string             Class name as a Python string
     #  bases        ExprNode           Base class tuple
     #  dict         ExprNode           Class dict (not owned by this node)
     #  doc          ExprNode or None   Doc string
     #  module_name  string             Name of defining module
     
-    subexprs = ['name', 'bases', 'doc']
-    
+    subexprs = ['bases', 'doc']
+
     def analyse_types(self, env):
-        self.name.analyse_types(env)
-        self.name = self.name.coerce_to_pyobject(env)
+        self.cname = env.intern_identifier(self.name)
         self.bases.analyse_types(env)
         if self.doc:
             self.doc.analyse_types(env)
@@ -2487,7 +2457,7 @@ class ClassNode(ExprNode):
         self.type = py_object_type
         self.is_temp = 1
         env.use_utility_code(create_class_utility_code);
-    
+
     def generate_result_code(self, code):
         if self.doc:
             code.put_error_if_neg(self.pos, 
@@ -2499,7 +2469,7 @@ class ClassNode(ExprNode):
                 self.result_code,
                 self.bases.py_result(),
                 self.dict.py_result(),
-                self.name.py_result(),
+                self.cname,
                 self.module_name,
                 code.error_goto_if_null(self.result_code, self.pos)))
 
@@ -3907,19 +3877,6 @@ class CloneNode(CoercionNode):
 #
 #------------------------------------------------------------------------------------
 
-get_name_utility_code = [
-"""
-static PyObject *__Pyx_GetName(PyObject *dict, char *name); /*proto*/
-""","""
-static PyObject *__Pyx_GetName(PyObject *dict, char *name) {
-    PyObject *result;
-    result = PyObject_GetAttrString(dict, name);
-    if (!result)
-        PyErr_SetString(PyExc_NameError, name);
-    return result;
-}
-"""]
-
 get_name_interned_utility_code = [
 """
 static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/
@@ -4069,7 +4026,7 @@ static int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
     if (obj == Py_None || PyObject_TypeCheck(obj, type))
         return 1;
     PyErr_Format(PyExc_TypeError, "Cannot convert %s to %s",
-        obj->ob_type->tp_name, type->tp_name);
+        Py_TYPE(obj)->tp_name, type->tp_name);
     return 0;
 }
 """]
@@ -4085,13 +4042,21 @@ static PyObject *__Pyx_CreateClass(
 {
     PyObject *py_modname;
     PyObject *result = 0;
-    
+
+    #if PY_MAJOR_VERSION < 3
     py_modname = PyString_FromString(modname);
+    #else
+    py_modname = PyUnicode_FromString(modname);
+    #endif
     if (!py_modname)
         goto bad;
     if (PyDict_SetItemString(dict, "__module__", py_modname) < 0)
         goto bad;
+    #if PY_MAJOR_VERSION < 3
     result = PyClass_New(bases, dict, name);
+    #else
+    result = PyObject_CallFunctionObjArgs((PyObject *)&PyType_Type, name, bases, dict, NULL);
+    #endif
 bad:
     Py_XDECREF(py_modname);
     return result;
@@ -4139,3 +4104,39 @@ static INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) {
 }
 """,""
 ]
+
+#------------------------------------------------------------------------------------
+
+type_cache_invalidation_code = [
+"""
+#if PY_VERSION_HEX >= 0x02060000
+static void __Pyx_TypeModified(PyTypeObject* type); /*proto*/
+#else
+  #define __Pyx_TypeModified(t)
+#endif
+""","""
+#if PY_VERSION_HEX >= 0x02060000
+/* copied from typeobject.c in Python 3.0a5 */
+static void __Pyx_TypeModified(PyTypeObject* type) {
+    PyObject *raw, *ref;
+    Py_ssize_t i, n;
+
+    if (!PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
+        return;
+
+    raw = type->tp_subclasses;
+    if (raw != NULL) {
+        n = PyList_GET_SIZE(raw);
+        for (i = 0; i < n; i++) {
+            ref = PyList_GET_ITEM(raw, i);
+            ref = PyWeakref_GET_OBJECT(ref);
+            if (ref != Py_None) {
+                __Pyx_TypeModified((PyTypeObject *)ref);
+            }
+        }
+    }
+    type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
+}
+#endif
+"""
+]

Cython/Compiler/Future.py

+def _get_feature(name):
+    import __future__
+    try:
+        return getattr(__future__, name)
+    except AttributeError:
+        # unique fake object for earlier Python versions
+        return object()
+
+unicode_literals = _get_feature("unicode_literals")
+
+del _get_feature

Cython/Compiler/Lexicon.py

@@ -5,7 +5,8 @@
 #   to be rebuilt next time pyrexc is run.
 #
 
-string_prefixes = "cCrRuU"
+raw_prefixes = "rR"
+string_prefixes = "cCuUbB"
 
 def make_lexicon():
     from Cython.Plex import \
@@ -55,9 +56,8 @@ def make_lexicon():
         + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq)) 
         + Str('"""')
     )
-    stringlit = Opt(Any(string_prefixes)) + (sq_string | dq_string | tsq_string| tdq_string)
     
-    beginstring = Opt(Any(string_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+    beginstring = Opt(Any(string_prefixes)) + Opt(Any(raw_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
     two_oct = octdigit + octdigit
     three_oct = octdigit + octdigit + octdigit
     two_hex = hexdigit + hexdigit

Cython/Compiler/Main.py

@@ -7,6 +7,12 @@ if sys.version_info[:2] < (2, 2):
     sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n")
     sys.exit(1)
 
+try:
+    set
+except NameError:
+    # Python 2.3
+    from sets import Set as set
+
 from time import time
 import Version
 from Scanning import PyrexScanner, FileSourceDescriptor
@@ -45,12 +51,14 @@ class Context:
     #
     #  modules               {string : ModuleScope}
     #  include_directories   [string]
+    #  future_directives     [object]
     
     def __init__(self, include_directories):
         #self.modules = {"__builtin__" : BuiltinScope()}
         import Builtin
         self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.include_directories = include_directories
+        self.future_directives = set()
         
     def find_module(self, module_name, 
             relative_to = None, pos = None, need_pxd = 1):

Cython/Compiler/ModuleNode.py

@@ -20,7 +20,7 @@ import PyrexTypes
 import TypeSlots
 import Version
 
-from Errors import error
+from Errors import error, warning
 from PyrexTypes import py_object_type
 from Cython.Utils import open_new_file, replace_suffix
 
@@ -221,12 +221,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("/* Implementation of %s */" % env.qualified_name)
         self.generate_const_definitions(env, code)
         self.generate_interned_num_decls(env, code)
-        self.generate_interned_name_decls(env, code)
+        self.generate_interned_string_decls(env, code)
         self.generate_py_string_decls(env, code)
         self.generate_cached_builtins_decls(env, code)
         self.body.generate_function_definitions(env, code, options.transforms)
         code.mark_pos(None)
-        self.generate_interned_name_table(env, code)
         self.generate_py_string_table(env, code)
         self.generate_typeobj_definitions(env, code)
         self.generate_method_table(env, code)
@@ -366,6 +365,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("#ifndef PY_LONG_LONG")
         code.putln("  #define PY_LONG_LONG LONG_LONG")
         code.putln("#endif")
+        code.putln("#ifndef DL_EXPORT")
+        code.putln("  #define DL_EXPORT(t) t")
+        code.putln("#endif")
+        code.putln("#if PY_VERSION_HEX < 0x02040000")
+        code.putln("  #define METH_COEXIST 0")
+        code.putln("#endif")
+
         code.putln("#if PY_VERSION_HEX < 0x02050000")
         code.putln("  typedef int Py_ssize_t;")
         code.putln("  #define PY_SSIZE_T_MAX INT_MAX")
@@ -375,9 +381,68 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define PyNumber_Index(o)    PyNumber_Int(o)")
         code.putln("  #define PyIndex_Check(o)     PyNumber_Check(o)")
         code.putln("#endif")
-        code.putln("#if PY_VERSION_HEX < 0x02040000")
-        code.putln("  #define METH_COEXIST 0")
+
+        code.putln("#if PY_VERSION_HEX < 0x02060000")
+        code.putln("  #define Py_REFCNT(ob) (((PyObject*)(ob))->ob_refcnt)")
+        code.putln("  #define Py_TYPE(ob)   (((PyObject*)(ob))->ob_type)")
+        code.putln("  #define Py_SIZE(ob)   ((PyVarObject*)(ob))->ob_size)")
+        code.putln("  #define PyVarObject_HEAD_INIT(type, size) \\")
+        code.putln("          PyObject_HEAD_INIT(type) size,")
+        code.putln("")
+        code.putln("  typedef struct {")
+        code.putln("     void *buf;")
+        code.putln("     Py_ssize_t len;")
+        code.putln("     int readonly;")
+        code.putln("     const char *format;")
+        code.putln("     int ndim;")
+        code.putln("     Py_ssize_t *shape;")
+        code.putln("     Py_ssize_t *strides;")
+        code.putln("     Py_ssize_t *suboffsets;")
+        code.putln("     Py_ssize_t itemsize;")
+        code.putln("     void *internal;")
+        code.putln("  } Py_buffer;")
+        code.putln("")
+        code.putln("  #define PyBUF_SIMPLE 0")
+        code.putln("  #define PyBUF_WRITABLE 0x0001")
+        code.putln("  #define PyBUF_LOCK 0x0002")
+        code.putln("  #define PyBUF_FORMAT 0x0004")
+        code.putln("  #define PyBUF_ND 0x0008")
+        code.putln("  #define PyBUF_STRIDES (0x0010 | PyBUF_ND)")
+        code.putln("  #define PyBUF_C_CONTIGUOUS (0x0020 | PyBUF_STRIDES)")
+        code.putln("  #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)")
+        code.putln("  #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)")
+        code.putln("  #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)")
+        code.putln("#endif")
+
+        code.put(builtin_module_name_utility_code[0])
+
+        code.putln("#if PY_MAJOR_VERSION >= 3")
+        code.putln("  #define Py_TPFLAGS_CHECKTYPES 0")
+        code.putln("  #define Py_TPFLAGS_HAVE_INDEX 0")
+        code.putln("#endif")
+
+        code.putln("#if PY_MAJOR_VERSION >= 3")
+        code.putln("  #define PyBaseString_Type            PyUnicode_Type")
+        code.putln("  #define PyInt_Type                   PyLong_Type")
+        code.putln("  #define PyInt_Check(op)              PyLong_Check(op)")
+        code.putln("  #define PyInt_CheckExact(op)         PyLong_CheckExact(op)")
+        code.putln("  #define PyInt_FromString             PyLong_FromString")
+        code.putln("  #define PyInt_FromUnicode            PyLong_FromUnicode")
+        code.putln("  #define PyInt_FromLong               PyLong_FromLong")
+        code.putln("  #define PyInt_FromSize_t             PyLong_FromSize_t")
+        code.putln("  #define PyInt_FromSsize_t            PyLong_FromSsize_t")
+        code.putln("  #define PyInt_AsLong                 PyLong_AsLong")
+        code.putln("  #define PyInt_AS_LONG                PyLong_AS_LONG")
+        code.putln("  #define PyInt_AsSsize_t              PyLong_AsSsize_t")
+        code.putln("  #define PyInt_AsUnsignedLongMask     PyLong_AsUnsignedLongMask")
+        code.putln("  #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask")
+        code.putln("  #define PyNumber_Divide(x,y)         PyNumber_TrueDivide(x,y)")
         code.putln("#endif")
+
+        code.putln("#if PY_MAJOR_VERSION >= 3")
+        code.putln("  #define PyMethod_New(func, self, klass) PyInstanceMethod_New(func)")
+        code.putln("#endif")
+
         code.putln("#ifndef __stdcall")
         code.putln("  #define __stdcall")
         code.putln("#endif")
@@ -664,6 +729,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     if scope.defines_any(["__setitem__", "__delitem__"]):
                         self.generate_ass_subscript_function(scope, code)
                     if scope.defines_any(["__setslice__", "__delslice__"]):
+                        warning(self.pos, "__setslice__ and __delslice__ are not supported by Python 3")
                         self.generate_ass_slice_function(scope, code)
                     if scope.defines_any(["__getattr__","__getattribute__"]):
                         self.generate_getattro_function(scope, code)
@@ -786,7 +852,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     "%s(o);" % tp_dealloc)
         else:
             code.putln(
-                    "(*o->ob_type->tp_free)(o);")
+                    "(*Py_TYPE(o)->tp_free)(o);")
         code.putln(
             "}")
     
@@ -800,14 +866,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(
                     "PyErr_Fetch(&etype, &eval, &etb);")
             code.putln(
-                    "++o->ob_refcnt;")
+                    "++Py_REFCNT(o);")
             code.putln(
                     "%s(o);" % 
                         entry.func_cname)
             code.putln(
                     "if (PyErr_Occurred()) PyErr_WriteUnraisable(o);")
             code.putln(
-                    "--o->ob_refcnt;")
+                    "--Py_REFCNT(o);")
             code.putln(
                     "PyErr_Restore(etype, eval, etb);")
             code.putln(
@@ -905,7 +971,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln(
                 "PyObject *x = PyInt_FromSsize_t(i); if(!x) return 0;")
         code.putln(
-                "r = o->ob_type->tp_as_mapping->mp_subscript(o, x);")
+                "r = Py_TYPE(o)->tp_as_mapping->mp_subscript(o, x);")
         code.putln(
                 "Py_DECREF(x);")
         code.putln(
@@ -936,7 +1002,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(
                     "PyErr_Format(PyExc_NotImplementedError,")
             code.putln(
-                    '  "Subscript assignment not supported by %s", o->ob_type->tp_name);')
+                    '  "Subscript assignment not supported by %s", Py_TYPE(o)->tp_name);')
             code.putln(
                     "return -1;")
         code.putln(
@@ -953,7 +1019,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(
                     "PyErr_Format(PyExc_NotImplementedError,")
             code.putln(
-                    '  "Subscript deletion not supported by %s", o->ob_type->tp_name);')
+                    '  "Subscript deletion not supported by %s", Py_TYPE(o)->tp_name);')
             code.putln(
                     "return -1;")
         code.putln(
@@ -1003,7 +1069,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(
                     "PyErr_Format(PyExc_NotImplementedError,")
             code.putln(
-                    '  "2-element slice assignment not supported by %s", o->ob_type->tp_name);')
+                    '  "2-element slice assignment not supported by %s", Py_TYPE(o)->tp_name);')
             code.putln(
                     "return -1;")
         code.putln(
@@ -1020,7 +1086,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln(
                     "PyErr_Format(PyExc_NotImplementedError,")
             code.putln(
-                    '  "2-element slice deletion not supported by %s", o->ob_type->tp_name);')
+                    '  "2-element slice deletion not supported by %s", Py_TYPE(o)->tp_name);')
             code.putln(
                     "return -1;")
         code.putln(
@@ -1261,9 +1327,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         #code.putln(header % scope.parent_type.typeobj_cname)
         code.putln(header % type.typeobj_cname)
         code.putln(
-            "PyObject_HEAD_INIT(0)")
-        code.putln(
-            "0, /*ob_size*/")
+            "PyVarObject_HEAD_INIT(0, 0)")
         code.putln(
             '"%s.%s", /*tp_name*/' % (
                 self.full_module_name, scope.class_name))
@@ -1339,26 +1403,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     "{0, 0, 0, 0, 0}")
             code.putln(
                 "};")
-    
-    def generate_interned_name_table(self, env, code):
-        code.mark_pos(None)
-        items = env.intern_map.items()
-        if items:
-            items.sort()
-            code.putln("")
-            code.putln(
-                "static __Pyx_InternTabEntry %s[] = {" %
-                    Naming.intern_tab_cname)
-            for (name, cname) in items:
-                code.putln(
-                    '{&%s, "%s"},' % (
-                        cname,
-                        name))
-            code.putln(
-                "{0, 0}")
-            code.putln(
-                "};")
-    
+
     def generate_py_string_table(self, env, code):
         entries = env.all_pystring_entries
         if entries:
@@ -1368,18 +1413,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     Naming.stringtab_cname)
             for entry in entries:
                 code.putln(
-                    "{&%s, %s, sizeof(%s), %d}," % (
+                    "{&%s, %s, sizeof(%s), %d, %d, %d}," % (
                         entry.pystring_cname,
                         entry.cname,
                         entry.cname,
-                        entry.type.is_unicode
+                        entry.type.is_unicode,
+                        entry.is_interned,
+                        entry.is_identifier
                         ))
             code.putln(
-                "{0, 0, 0, 0}")
+                "{0, 0, 0, 0, 0}")
             code.putln(
                 "};")
 
-
     def generate_filename_init_prototype(self, code):
         code.putln("");
         code.putln("static void %s(void); /*proto*/" % Naming.fileinit_cname)
@@ -1390,6 +1436,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("%s; /*proto*/" % header)
         code.putln("%s {" % header)
         code.put_var_declarations(env.temp_entries)
+        code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)));
 
         code.putln("/*--- Libary function declarations ---*/")
         env.generate_library_function_declarations(code)
@@ -1408,7 +1455,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln("/*--- Builtin init code ---*/")
             self.generate_builtin_init_code(env, code)
             
-        code.putln("%s = PyTuple_New(0); %s" % (Naming.empty_tuple, code.error_goto_if_null(Naming.empty_tuple, self.pos)));
         code.putln("%s = 0;" % Naming.skip_dispatch_cname);
 
         code.putln("/*--- Global init code ---*/")
@@ -1469,9 +1515,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("/*--- Intern cleanup code ---*/")
         for entry in env.pynum_entries:
             code.put_var_decref_clear(entry)
-        if env.intern_map:
-            for name, cname in env.intern_map.items():
-                code.put_decref_clear(cname, PyrexTypes.py_object_type)
+        if env.all_pystring_entries:
+            for entry in env.all_pystring_entries:
+                if entry.is_interned:
+                    code.put_decref_clear(
+                        entry.pystring_cname, PyrexTypes.py_object_type)
         code.putln("Py_INCREF(Py_None); return Py_None;")
         code.putln('}')
 
@@ -1496,7 +1544,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 env.module_cname,
                 code.error_goto(self.pos)));
         code.putln(
-            '%s = PyImport_AddModule("__builtin__");' %
+            '%s = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME);' %
                 Naming.builtins_cname)
         code.putln(
             "if (!%s) %s;" % (
@@ -1523,12 +1571,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 entry.cname,
                 entry.init,
                 code.error_goto_if_null(entry.cname, self.pos)))
-        if env.intern_map:
-            env.use_utility_code(Nodes.init_intern_tab_utility_code);
-            code.putln(
-                "if (__Pyx_InternStrings(%s) < 0) %s;" % (
-                    Naming.intern_tab_cname,
-                    code.error_goto(self.pos)))
     
     def generate_string_init_code(self, env, code):
         if env.all_pystring_entries:
@@ -1542,23 +1584,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         # Lookup and cache builtin objects.
         if Options.cache_builtins:
             for entry in env.cached_builtins:
-                if Options.intern_names:
-                    #assert entry.interned_cname is not None
-                    code.putln(
-                        '%s = __Pyx_GetName(%s, %s); if (!%s) %s' % (
-                        entry.cname,
-                        Naming.builtins_cname,
-                        entry.interned_cname,
-                        entry.cname,
-                        code.error_goto(entry.pos)))
-                else:
-                    code.putln(
-                        '%s = __Pyx_GetName(%s, "%s"); if (!%s) %s' % (
-                        entry.cname,
-                        Naming.builtins_cname,
-                        entry.name,
-                        entry.cname, 
-                        code.error_goto(entry.pos)))
+                #assert entry.interned_cname is not None
+                code.putln(
+                    '%s = __Pyx_GetName(%s, %s); if (!%s) %s' % (
+                    entry.cname,
+                    Naming.builtins_cname,
+                    entry.interned_cname,
+                    entry.cname,
+                    code.error_goto(entry.pos)))
     
     def generate_global_init_code(self, env, code):
         # Generate code to initialise global PyObject *
@@ -1657,17 +1690,35 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             env.use_utility_code(Nodes.get_vtable_utility_code)
         env.types_imported[type] = 1
 
+    py3_type_name_map = {'str' : 'bytes', 'unicode' : 'str'}
+
     def generate_type_import_call(self, type, code, error_code):
         if type.typedef_flag:
             objstruct = type.objstruct_cname
         else:
             objstruct = "struct %s" % type.objstruct_cname
-        code.putln('%s = __Pyx_ImportType("%s", "%s", sizeof(%s)); %s' % (
-            type.typeptr_cname,
-            type.module_name, 
-            type.name,
-            objstruct,
-            error_code))
+        module_name = type.module_name
+        if module_name not in ('__builtin__', 'builtins'):
+            module_name = '"%s"' % module_name
+        else:
+            module_name = '__Pyx_BUILTIN_MODULE_NAME'
+        if type.name in self.py3_type_name_map:
+            code.putln("#if PY_MAJOR_VERSION >= 3")
+            code.putln('%s = __Pyx_ImportType(%s, "%s", sizeof(%s)); %s' % (
+                    type.typeptr_cname,
+                    module_name,
+                    self.py3_type_name_map[type.name],
+                    objstruct,
+                    error_code))
+            code.putln("#else")
+        code.putln('%s = __Pyx_ImportType(%s, "%s", sizeof(%s)); %s' % (
+                type.typeptr_cname,
+                module_name,
+                type.name,
+                objstruct,
+                error_code))
+        if type.name in self.py3_type_name_map:
+            code.putln("#endif")
 
     def generate_type_ready_code(self, env, entry, code):
         # Generate a call to PyType_Ready for an extension
@@ -1759,6 +1810,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 #
 #------------------------------------------------------------------------------------
 
+builtin_module_name_utility_code = [
+"""\
+#if PY_MAJOR_VERSION < 3
+  #define __Pyx_BUILTIN_MODULE_NAME "__builtin__"
+#else
+  #define __Pyx_BUILTIN_MODULE_NAME "builtins"
+#endif
+"""]
+
+
 import_module_utility_code = [
 """
 static PyObject *__Pyx_ImportModule(char *name); /*proto*/
@@ -1768,8 +1829,12 @@ static PyObject *__Pyx_ImportModule(char *name); /*proto*/
 static PyObject *__Pyx_ImportModule(char *name) {
     PyObject *py_name = 0;
     PyObject *py_module = 0;
-    
+
+    #if PY_MAJOR_VERSION < 3
     py_name = PyString_FromString(name);
+    #else
+    py_name = PyUnicode_FromString(name);
+    #endif
     if (!py_name)
         goto bad;
     py_module = PyImport_Import(py_name);
@@ -1796,8 +1861,12 @@ static PyTypeObject *__Pyx_ImportType(char *module_name, char *class_name,
     PyObject *py_module = 0;
     PyObject *result = 0;
     PyObject *py_name = 0;
-    
+
+    #if PY_MAJOR_VERSION < 3
     py_name = PyString_FromString(module_name);
+    #else
+    py_name = PyUnicode_FromString(module_name);
+    #endif
     if (!py_name)
         goto bad;
 

Cython/Compiler/Naming.py

@@ -62,6 +62,8 @@ c_api_tab_cname  = pyrex_prefix + "c_api_tab"
 gilstate_cname   = pyrex_prefix + "state"
 skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
 empty_tuple      = pyrex_prefix + "empty_tuple"
+print_function   = pyrex_prefix + "print"
+print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
 optional_args_cname = pyrex_prefix + "optional_args"
 no_opt_args      = pyrex_prefix + "no_opt_args"

Cython/Compiler/Nodes.py

@@ -245,28 +245,28 @@ class BlockNode:
             for entry in env.const_entries:
                 if not entry.is_interned:
                     code.put_var_declaration(entry, static = 1)
-    
-    def generate_interned_name_decls(self, env, code):
-        #  Flush accumulated interned names from the global scope
-        #  and generate declarations for them.
-        genv = env.global_scope()
-        intern_map = genv.intern_map
-        names = genv.interned_names
-        if names:
+
+    def generate_interned_string_decls(self, env, code):
+        entries = env.global_scope().new_interned_string_entries
+        if entries:
+            code.putln("")
+            for entry in entries:
+                code.put_var_declaration(entry, static = 1)
             code.putln("")
-            for name in names:
+            for entry in entries:
                 code.putln(
-                    "static PyObject *%s;" % intern_map[name])
-            del names[:]
-    
+                    "static PyObject *%s;" % entry.pystring_cname)
+            del entries[:]
+
     def generate_py_string_decls(self, env, code):
         entries = env.pystring_entries
         if entries:
             code.putln("")
             for entry in entries:
-                code.putln(
-                    "static PyObject *%s;" % entry.pystring_cname)
-    
+                if not entry.is_interned:
+                    code.putln(
+                        "static PyObject *%s;" % entry.pystring_cname)
+
     def generate_interned_num_decls(self, env, code):
         #  Flush accumulated interned nums from the global scope
         #  and generate declarations for them.
@@ -893,7 +893,7 @@ class FuncDefNode(StatNode, BlockNode):
         # if we supported them, which we probably won't.
         # ----- Top-level constants used by this function
         self.generate_interned_num_decls(lenv, code)
-        self.generate_interned_name_decls(lenv, code)
+        self.generate_interned_string_decls(lenv, code)
         self.generate_py_string_decls(lenv, code)
         self.generate_cached_builtins_decls(lenv, code)
         #code.putln("")
@@ -1100,8 +1100,8 @@ class CFuncDefNode(FuncDefNode):
             self.entry.as_variable = self.py_func.entry
             # Reset scope entry the above cfunction
             env.entries[name] = self.entry
-            if Options.intern_names:
-                self.py_func.interned_attr_cname = env.intern(self.py_func.entry.name)
+            self.py_func.interned_attr_cname = env.intern_identifier(
+                self.py_func.entry.name)
             if not env.is_module_scope or Options.lookup_module_cpdef:
                 self.override = OverrideCheckNode(self.pos, py_func = self.py_func)
                 self.body = StatListNode(self.pos, stats=[self.override, self.body])
@@ -1938,15 +1938,12 @@ class OverrideCheckNode(StatNode):
         if self.py_func.is_module_scope:
             code.putln("else {")
         else:
-            code.putln("else if (unlikely(%s->ob_type->tp_dictoffset != 0)) {" % self_arg)
+            code.putln("else if (unlikely(Py_TYPE(%s)->tp_dictoffset != 0)) {" % self_arg)
         err = code.error_goto_if_null(self_arg, self.pos)
         # need to get attribute manually--scope would return cdef method
-        if Options.intern_names:
-            code.putln("%s = PyObject_GetAttr(%s, %s); %s" % (self.func_node.result_code, self_arg, self.py_func.interned_attr_cname, err))
-        else:
-            code.putln('%s = PyObject_GetAttrString(%s, "%s"); %s' % (self.func_node.result_code, self_arg, self.py_func.entry.name, err))
+        code.putln("%s = PyObject_GetAttr(%s, %s); %s" % (self.func_node.result_code, self_arg, self.py_func.interned_attr_cname, err))
         # It appears that this type is not anywhere exposed in the Python/C API
-        is_builtin_function_or_method = '(strcmp(%s->ob_type->tp_name, "builtin_function_or_method") == 0)' % self.func_node.result_code
+        is_builtin_function_or_method = '(strcmp(Py_TYPE(%s)->tp_name, "builtin_function_or_method") == 0)' % self.func_node.result_code
         is_overridden = '(PyCFunction_GET_FUNCTION(%s) != (void *)&%s)' % (self.func_node.result_code, self.py_func.entry.func_cname)
         code.putln('if (!%s || %s) {' % (is_builtin_function_or_method, is_overridden))
         self.body.generate_execution_code(code)
@@ -1985,8 +1982,7 @@ class PyClassDefNode(StatNode, BlockNode):
             doc_node = ExprNodes.StringNode(pos, value = doc)
         else:
             doc_node = None
-        self.classobj = ExprNodes.ClassNode(pos,
-            name = ExprNodes.StringNode(pos, value = name), 
+        self.classobj = ExprNodes.ClassNode(pos, name = name,
             bases = bases, dict = self.dict, doc = doc_node)
         self.target = ExprNodes.NameNode(pos, name = name)
     
@@ -2139,10 +2135,11 @@ class PropertyNode(StatNode):
         entry = env.declare_property(self.name, self.doc, self.pos)
         if entry:
             if self.doc and Options.docstrings:
-                doc_entry = env.get_string_const(self.doc)
+                doc_entry = env.get_string_const(
+                    self.doc, identifier = False)
                 entry.doc_cname = doc_entry.cname
             self.body.analyse_declarations(entry.scope)
-        
+
     def analyse_expressions(self, env):
         self.body.analyse_expressions(env)
     
@@ -2513,12 +2510,17 @@ class InPlaceAssignmentNode(AssignmentNode):
 class PrintStatNode(StatNode):
     #  print statement
     #
-    #  args              [ExprNode]
-    #  ends_with_comma   boolean
-    
-    child_attrs = ["args"]
-    
+    #  arg_tuple         TupleNode
+    #  append_newline    boolean
+
+    child_attrs = ["arg_tuple"]
+
     def analyse_expressions(self, env):
+        self.arg_tuple.analyse_expressions(env)
+        self.arg_tuple = self.arg_tuple.coerce_to_pyobject(env)
+        self.arg_tuple.release_temp(env)
+        env.use_utility_code(printing_utility_code)
+        return
         for i in range(len(self.args)):
             arg = self.args[i]
             arg.analyse_types(env)
@@ -2527,24 +2529,18 @@ class PrintStatNode(StatNode):
             arg.release_temp(env)
             self.args[i] = arg
             #env.recycle_pending_temps() # TEMPORARY
-        env.use_utility_code(printing_utility_code)
-    
+
     def generate_execution_code(self, code):
-        for arg in self.args:
-            arg.generate_evaluation_code(code)
-            code.putln(
-                "if (__Pyx_PrintItem(%s) < 0) %s" % (
-                    arg.py_result(),
-                    code.error_goto(self.pos)))
-            arg.generate_disposal_code(code)
-        if not self.ends_with_comma:
-            code.putln(
-                "if (__Pyx_PrintNewline() < 0) %s" %
-                    code.error_goto(self.pos))
-                    
+        self.arg_tuple.generate_evaluation_code(code)
+        code.putln(
+            "if (__Pyx_Print(%s, %d) < 0) %s" % (
+                self.arg_tuple.py_result(),
+                self.append_newline,
+                code.error_goto(self.pos)))
+        self.arg_tuple.generate_disposal_code(code)
+
     def annotate(self, code):
-        for arg in self.args:
-            arg.annotate(code)
+        self.arg_tuple.annotate(code)
 
 
 class DelStatNode(StatNode):
@@ -3005,15 +3001,16 @@ class ForInStatNode(LoopNode, StatNode):
         else:
             step = args[2]
             if isinstance(step, ExprNodes.UnaryMinusNode) and isinstance(step.operand, ExprNodes.IntNode):
-                step = ExprNodes.IntNode(pos = step.pos, value=-int(step.operand.value))
+                step = ExprNodes.IntNode(pos = step.pos, value=str(-int(step.operand.value, 0)))
             if isinstance(step, ExprNodes.IntNode):
-                if step.value > 0:
+                step_value = int(step.value, 0)
+                if step_value > 0:
                     self.step = step
                     self.relation1 = '<='
                     self.relation2 = '<'
                     return True
-                elif step.value < 0:
-                    self.step = ExprNodes.IntNode(pos = step.pos, value=-int(step.value))
+                elif step_value < 0:
+                    self.step = ExprNodes.IntNode(pos = step.pos, value=str(-step_value))
                     self.relation1 = '>='
                     self.relation2 = '>'
                     return True
@@ -3628,7 +3625,7 @@ class CImportStatNode(StatNode):
             return
         module_scope = env.find_module(self.module_name, self.pos)
         if "." in self.module_name:
-            names = self.module_name.split(".")
+            names = [EncodedString(name) for name in self.module_name.split(".")]
             top_name = names[0]
             top_module_scope = env.context.find_submodule(top_name)
             module_scope = top_module_scope
@@ -3699,8 +3696,8 @@ class FromImportStatNode(StatNode):
         self.item.allocate_temp(env)
         self.interned_items = []
         for name, target in self.items:
-            if Options.intern_names:
-                self.interned_items.append((env.intern(name), target))
+            self.interned_items.append(
+                (env.intern_identifier(name), target))
             target.analyse_target_expression(env, None)
             #target.release_target_temp(env) # was release_temp ?!?
         self.module.release_temp(env)
@@ -3708,24 +3705,14 @@ class FromImportStatNode(StatNode):
     
     def generate_execution_code(self, code):
         self.module.generate_evaluation_code(code)
-        if Options.intern_names:
-            for cname, target in self.interned_items:
-                code.putln(
-                    '%s = PyObject_GetAttr(%s, %s); %s' % (
-                        self.item.result_code, 
-                        self.module.py_result(),
-                        cname,
-                        code.error_goto_if_null(self.item.result_code, self.pos)))
-                target.generate_assignment_code(self.item, code)
-        else:
-            for name, target in self.items:
-                code.putln(
-                    '%s = PyObject_GetAttrString(%s, "%s"); %s' % (
-                        self.item.result_code, 
-                        self.module.py_result(),
-                        name,
-                        code.error_goto_if_null(self.item.result_code, self.pos)))
-                target.generate_assignment_code(self.item, code)
+        for cname, target in self.interned_items:
+            code.putln(
+                '%s = PyObject_GetAttr(%s, %s); %s' % (
+                    self.item.result_code, 
+                    self.module.py_result(),
+                    cname,
+                    code.error_goto_if_null(self.item.result_code, self.pos)))
+            target.generate_assignment_code(self.item, code)
         self.module.generate_disposal_code(code)
 
 #------------------------------------------------------------------------------------
@@ -3744,8 +3731,7 @@ utility_function_predeclarations = \
 #define INLINE 
 #endif
 
-typedef struct {PyObject **p; char *s;} __Pyx_InternTabEntry; /*proto*/
-typedef struct {PyObject **p; char *s; long n; int is_unicode;} __Pyx_StringTabEntry; /*proto*/
+typedef struct {PyObject **p; char *s; long n; char is_unicode; char intern; char is_identifier;} __Pyx_StringTabEntry; /*proto*/
 
 """ + """
 
@@ -3788,9 +3774,13 @@ else:
 
 printing_utility_code = [
 """
-static int __Pyx_PrintItem(PyObject *); /*proto*/
-static int __Pyx_PrintNewline(void); /*proto*/
-""",r"""
+static int __Pyx_Print(PyObject *, int); /*proto*/
+#if PY_MAJOR_VERSION >= 3
+static PyObject* %s = 0;
+static PyObject* %s = 0;
+#endif
+""" % (Naming.print_function, Naming.print_function_kwargs), r"""
+#if PY_MAJOR_VERSION < 3
 static PyObject *__Pyx_GetStdout(void) {
     PyObject *f = PySys_GetObject("stdout");
     if (!f) {
@@ -3799,39 +3789,75 @@ static PyObject *__Pyx_GetStdout(void) {
     return f;
 }
 
-static int __Pyx_PrintItem(PyObject *v) {
+static int __Pyx_Print(PyObject *arg_tuple, int newline) {
     PyObject *f;
+    PyObject* v;
+    int i;
     
     if (!(f = __Pyx_GetStdout()))
         return -1;
-    if (PyFile_SoftSpace(f, 1)) {
-        if (PyFile_WriteString(" ", f) < 0)
+    for (i=0; i < PyTuple_GET_SIZE(arg_tuple); i++) {
+        if (PyFile_SoftSpace(f, 1)) {
+            if (PyFile_WriteString(" ", f) < 0)
+                return -1;
+        }
+        v = PyTuple_GET_ITEM(arg_tuple, i);
+        if (PyFile_WriteObject(v, f, Py_PRINT_RAW) < 0)
             return -1;
+        if (PyString_Check(v)) {
+            char *s = PyString_AsString(v);
+            Py_ssize_t len = PyString_Size(v);
+            if (len > 0 &&
+                isspace(Py_CHARMASK(s[len-1])) &&
+                s[len-1] != ' ')
+                    PyFile_SoftSpace(f, 0);
+        }
     }
-    if (PyFile_WriteObject(v, f, Py_PRINT_RAW) < 0)
-        return -1;
-    if (PyString_Check(v)) {
-        char *s = PyString_AsString(v);
-        Py_ssize_t len = PyString_Size(v);
-        if (len > 0 &&
-            isspace(Py_CHARMASK(s[len-1])) &&
-            s[len-1] != ' ')
-                PyFile_SoftSpace(f, 0);
+    if (newline) {
+        if (PyFile_WriteString("\n", f) < 0)
+            return -1;
+        PyFile_SoftSpace(f, 0);
     }
     return 0;
 }
 
-static int __Pyx_PrintNewline(void) {
-    PyObject *f;
-    
-    if (!(f = __Pyx_GetStdout()))
-        return -1;
-    if (PyFile_WriteString("\n", f) < 0)
+#else /* Python 3 has a print function */
+static int __Pyx_Print(PyObject *arg_tuple, int newline) {
+    PyObject* kwargs = 0;
+    PyObject* result = 0;
+    PyObject* end_string;
+    if (!%(PRINT_FUNCTION)s) {
+        %(PRINT_FUNCTION)s = PyObject_GetAttrString(%(BUILTINS)s, "print");
+        if (!%(PRINT_FUNCTION)s)
+            return -1;
+    }
+    if (!newline) {
+        if (!%(PRINT_KWARGS)s) {
+            %(PRINT_KWARGS)s = PyDict_New();
+            if (!%(PRINT_KWARGS)s)
+                return -1;
+            end_string = PyUnicode_FromStringAndSize(" ", 1);
+            if (!end_string)
+                return -1;
+            if (PyDict_SetItemString(%(PRINT_KWARGS)s, "end", end_string) < 0) {
+                Py_DECREF(end_string);
+                return -1;
+            }
+            Py_DECREF(end_string);
+        }
+        kwargs = %(PRINT_KWARGS)s;
+    }
+    result = PyObject_Call(%(PRINT_FUNCTION)s, arg_tuple, kwargs);
+    if (!result)
         return -1;
-    PyFile_SoftSpace(f, 0);
+    Py_DECREF(result);
     return 0;
 }
-"""]
+#endif
+""" % {'BUILTINS'       : Naming.builtins_cname,
+       'PRINT_FUNCTION' : Naming.print_function,
+       'PRINT_KWARGS'   : Naming.print_function_kwargs}
+]
 
 #------------------------------------------------------------------------------------
 
@@ -3887,7 +3913,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb) {
                 goto raise_error;
             }
         #else
-            type = (PyObject*) type->ob_type;
+            type = (PyObject*) Py_TYPE(type);
             Py_INCREF(type);
             if (!PyType_IsSubtype((PyTypeObject *)type, (PyTypeObject *)PyExc_BaseException)) {
                 PyErr_SetString(PyExc_TypeError,
@@ -3937,14 +3963,14 @@ static int __Pyx_ArgTypeTest(PyObject *obj, PyTypeObject *type, int none_allowed
     }
     if (none_allowed && obj == Py_None) return 1;
     else if (exact) {
-        if (obj->ob_type == type) return 1;
+        if (Py_TYPE(obj) == type) return 1;
     }
     else {
         if (PyObject_TypeCheck(obj, type)) return 1;
     }
     PyErr_Format(PyExc_TypeError,
         "Argument '%s' has incorrect type (expected %s, got %s)",
-        name, type->tp_name, obj->ob_type->tp_name);
+        name, type->tp_name, Py_TYPE(obj)->tp_name);
     return 0;
 }
 """]
@@ -4025,7 +4051,11 @@ static int __Pyx_CheckKeywordStrings(
     PyObject* key = 0;
     Py_ssize_t pos = 0;
     while (PyDict_Next(kwdict, &pos, &key, 0)) {
+        #if PY_MAJOR_VERSION < 3
         if (unlikely(!PyString_Check(key))) {
+        #else
+        if (unlikely(!PyUnicode_Check(key))) {
+        #endif
             PyErr_Format(PyExc_TypeError,
                          "%s() keywords must be strings", function_name);
             return 0;
@@ -4034,7 +4064,11 @@ static int __Pyx_CheckKeywordStrings(
     if (unlikely(!kw_allowed) && unlikely(key)) {
         PyErr_Format(PyExc_TypeError,
                      "'%s' is an invalid keyword argument for this function",
+        #if PY_MAJOR_VERSION < 3
                      PyString_AsString(key));
+        #else
+                     PyUnicode_AsString(key));
+        #endif
         return 0;
     }
     return 1;
@@ -4082,7 +4116,11 @@ static int __Pyx_SplitKeywords(
         if (!*kwds2)
             goto bad;
         for (i = 0, p = kwd_list; *p; i++, p++) {
+            #if PY_MAJOR_VERSION < 3
             s = PyString_FromString(*p);
+            #else
+            s = PyUnicode_FromString(*p);
+            #endif
             x = PyDict_GetItem(*kwds, s);
             if (x) {
                 if (PyDict_SetItem(kwds1, s, x) < 0)
@@ -4162,7 +4200,11 @@ static void __Pyx_WriteUnraisable(char *name) {
     PyObject *old_exc, *old_val, *old_tb;
     PyObject *ctx;
     PyErr_Fetch(&old_exc, &old_val, &old_tb);
+    #if PY_MAJOR_VERSION < 3
     ctx = PyString_FromString(name);
+    #else
+    ctx = PyUnicode_FromString(name);
+    #endif
     PyErr_Restore(old_exc, old_val, old_tb);
     if (!ctx)
         ctx = Py_None;
@@ -4187,22 +4229,37 @@ static void __Pyx_AddTraceback(char *funcname) {
     PyObject *empty_string = 0;
     PyCodeObject *py_code = 0;
     PyFrameObject *py_frame = 0;
-    
+
+    #if PY_MAJOR_VERSION < 3
     py_srcfile = PyString_FromString(%(FILENAME)s);
+    #else
+    py_srcfile = PyUnicode_FromString(%(FILENAME)s);
+    #endif
     if (!py_srcfile) goto bad;
     if (%(CLINENO)s) {
+        #if PY_MAJOR_VERSION < 3
         py_funcname = PyString_FromFormat( "%%s (%%s:%%u)", funcname, %(CFILENAME)s, %(CLINENO)s);
+        #else
+        py_funcname = PyUnicode_FromFormat( "%%s (%%s:%%u)", funcname, %(CFILENAME)s, %(CLINENO)s);
+        #endif
     }
     else {
+        #if PY_MAJOR_VERSION < 3
         py_funcname = PyString_FromString(funcname);
+        #else
+        py_funcname = PyUnicode_FromString(funcname);
+        #endif
     }
     if (!py_funcname) goto bad;
     py_globals = PyModule_GetDict(%(GLOBALS)s);
     if (!py_globals) goto bad;
-    empty_string = PyString_FromString("");
+    empty_string = PyString_FromStringAndSize("", 0);
     if (!empty_string) goto bad;
     py_code = PyCode_New(
         0,            /*int argcount,*/
+        #if PY_MAJOR_VERSION >= 3
+        0,            /*int kwonlyargcount,*/
+        #endif
         0,            /*int nlocals,*/
         0,            /*int stacksize,*/
         0,            /*int flags,*/
@@ -4298,32 +4355,26 @@ done:
 
 #------------------------------------------------------------------------------------
 
-init_intern_tab_utility_code = [
-"""
-static int __Pyx_InternStrings(__Pyx_InternTabEntry *t); /*proto*/
-""","""
-static int __Pyx_InternStrings(__Pyx_InternTabEntry *t) {
-    while (t->p) {
-        *t->p = PyString_InternFromString(t->s);
-        if (!*t->p)
-            return -1;
-        ++t;
-    }
-    return 0;
-}
-"""]
-
-#------------------------------------------------------------------------------------
-
 init_string_tab_utility_code = [
 """
 static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/
 ""","""
 static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
     while (t->p) {
-        if (t->is_unicode) {
+        #if PY_MAJOR_VERSION < 3
+        if (t->is_unicode && (!t->is_identifier)) {
             *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
-        } else {
+        } else if (t->intern) {
+            *t->p = PyString_InternFromString(t->s);
+        }
+        #else  /* Python 3+ has unicode identifiers */
+        if (t->is_identifier || (t->is_unicode && t->intern)) {
+            *t->p = PyUnicode_InternFromString(t->s);
+        } else if (t->is_unicode) {
+            *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
+        }
+        #endif
+        else {
             *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
         }
         if (!*t->p)

Cython/Compiler/Options.py

@@ -2,7 +2,6 @@
 #  Pyrex - Compilation-wide options
 #
 
-intern_names = 1    #  Intern global variable and attribute names
 cache_builtins = 1  #  Perform lookups on builtin names only once
 
 embed_pos_in_docstring = 0

Cython/Compiler/Parsing.py

@@ -11,6 +11,7 @@ import ExprNodes
 from ModuleNode import ModuleNode
 from Errors import error, InternalError
 from Cython import Utils
+import Future
 
 def p_ident(s, message = "Expected an identifier"):
     if s.sy == 'IDENT':
@@ -260,7 +261,7 @@ def p_trailer(s, node1):
         return p_index(s, node1)
     else: # s.sy == '.'
         s.next()
-        name = p_ident(s)
+        name = Utils.EncodedString( p_ident(s) )
         return ExprNodes.AttributeNode(pos, 
             obj = node1, attribute = name)
 
@@ -283,8 +284,7 @@ def p_call(s, function):
                 s.error("Expected an identifier before '='",
                     pos = arg.pos)
             encoded_name = Utils.EncodedString(arg.name)
-            encoded_name.encoding = s.source_encoding
-            keyword = ExprNodes.StringNode(arg.pos, 
+            keyword = ExprNodes.IdentifierStringNode(arg.pos, 
                 value = encoded_name)
             arg = p_simple_expr(s)
             keyword_args.append((keyword, arg))
@@ -469,7 +469,7 @@ def p_atom(s):
         else:
             return ExprNodes.StringNode(pos, value = value)
     elif sy == 'IDENT':
-        name = s.systring
+        name = Utils.EncodedString( s.systring )
         s.next()
         if name == "None":
             return ExprNodes.NoneNode(pos)
@@ -502,13 +502,8 @@ def p_name(s, name):
                 return ExprNodes.LongNode(pos, value = rep)
             elif isinstance(value, float):
                 return ExprNodes.FloatNode(pos, value = rep)
-            elif isinstance(value, str):
-                sval = Utils.EncodedString(rep[1:-1])
-                sval.encoding = value.encoding
-                return ExprNodes.StringNode(pos, value = sval)
             elif isinstance(value, unicode):
-                sval = Utils.EncodedString(rep[2:-1])
-                return ExprNodes.StringNode(pos, value = sval)
+                return ExprNodes.StringNode(pos, value = value)
             else:
                 error(pos, "Invalid type for compile-time constant: %s"
                     % value.__class__.__name__)
@@ -516,7 +511,7 @@ def p_name(s, name):
 
 def p_cat_string_literal(s):
     # A sequence of one or more adjacent string literals.
-    # Returns (kind, value) where kind in ('', 'c', 'r', 'u')
+    # Returns (kind, value) where kind in ('b', 'c', 'u')
     kind, value = p_string_literal(s)
     if kind != 'c':
         strings = [value]
@@ -541,13 +536,23 @@ def p_opt_string_literal(s):
 
 def p_string_literal(s):
     # A single string or char literal.
-    # Returns (kind, value) where kind in ('', 'c', 'r', 'u')
+    # Returns (kind, value) where kind in ('b', 'c', 'u')
     # s.sy == 'BEGIN_STRING'
     pos = s.position()
-    #is_raw = s.systring[:1].lower() == "r"
+    is_raw = 0
     kind = s.systring[:1].lower()
-    if kind not in "cru":
+    if kind == 'r':
+        kind = ''
+        is_raw = 1
+    elif kind in 'ub':
+        is_raw = s.systring[1:2].lower() == 'r'
+    elif kind != 'c':
         kind = ''
+    if Future.unicode_literals in s.context.future_directives:
+        if kind == '':
+            kind = 'u'
+    elif kind == '':
+        kind = 'b'
     chars = []
     while 1:
         s.next()
@@ -560,7 +565,7 @@ def p_string_literal(s):
             chars.append(systr)
         elif sy == 'ESCAPE':
             systr = s.systring
-            if kind == 'r':
+            if is_raw:
                 if systr == '\\\n':
                     chars.append(r'\\\n')
                 elif systr == r'\"':
@@ -823,17 +828,18 @@ def p_print_statement(s):
     if s.sy == '>>':
         s.error("'print >>' not yet implemented")
     args = []
-    ewc = 0
+    ends_with_comma = 0
     if s.sy not in ('NEWLINE', 'EOF'):
         args.append(p_simple_expr(s))
         while s.sy == ',':
             s.next()
             if s.sy in ('NEWLINE', 'EOF'):
-                ewc = 1
+                ends_with_comma = 1
                 break
             args.append(p_simple_expr(s))
-    return Nodes.PrintStatNode(pos, 
-        args = args, ends_with_comma = ewc)
+    arg_tuple = ExprNodes.TupleNode(pos, args = args)
+    return Nodes.PrintStatNode(pos,
+        arg_tuple = arg_tuple, append_newline = not ends_with_comma)
 
 def p_del_statement(s):
     # s.sy == 'del'
@@ -905,6 +911,7 @@ def p_import_statement(s):
         items.append(p_dotted_name(s, as_allowed = 1))
     stats = []
     for pos, target_name, dotted_name, as_name in items:
+        dotted_name = Utils.EncodedString(dotted_name)
         if kind == 'cimport':
             stat = Nodes.CImportStatNode(pos, 
                 module_name = dotted_name,
@@ -915,19 +922,17 @@ def p_import_statement(s):
                     ExprNodes.StringNode(pos, value = Utils.EncodedString("*"))])
             else:
                 name_list = None
-            dotted_name = Utils.EncodedString(dotted_name)
-            dotted_name.encoding = s.source_encoding
             stat = Nodes.SingleAssignmentNode(pos,
                 lhs = ExprNodes.NameNode(pos, 
                     name = as_name or target_name),
                 rhs = ExprNodes.ImportNode(pos, 
-                    module_name = ExprNodes.StringNode(pos,
-                        value = dotted_name),
+                    module_name = ExprNodes.IdentifierStringNode(
+                        pos, value = dotted_name),
                     name_list = name_list))
         stats.append(stat)
     return Nodes.StatListNode(pos, stats = stats)
 
-def p_from_import_statement(s):
+def p_from_import_statement(s, first_statement = 0):
     # s.sy == 'from'
     pos = s.position()
     s.next()
@@ -944,7 +949,20 @@ def p_from_import_statement(s):
     while s.sy == ',':
         s.next()
         imported_names.append(p_imported_name(s))
-    if kind == 'cimport':
+    dotted_name = Utils.EncodedString(dotted_name)
+    if dotted_name == '__future__':
+        if not first_statement:
+            s.error("from __future__ imports must occur at the beginning of the file")
+        else:
+            for (name_pos, name, as_name) in imported_names:
+                try:
+                    directive = getattr(Future, name)
+                except AttributeError:
+                    s.error("future feature %s is not defined" % name)
+                    break
+                s.context.future_directives.add(directive)
+        return Nodes.PassStatNode(pos)
+    elif kind == 'cimport':
         for (name_pos, name, as_name) in imported_names:
             local_name = as_name or name
             s.add_type_name(local_name)
@@ -956,9 +974,8 @@ def p_from_import_statement(s):
         items = []
         for (name_pos, name, as_name) in imported_names:
             encoded_name = Utils.EncodedString(name)
-            encoded_name.encoding = s.source_encoding
             imported_name_strings.append(
-                ExprNodes.StringNode(name_pos, value = encoded_name))
+                ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
             items.append(
                 (name,
                  ExprNodes.NameNode(name_pos, 
@@ -966,11 +983,9 @@ def p_from_import_statement(s):
         import_list = ExprNodes.ListNode(
             imported_names[0][0], args = imported_name_strings)
         dotted_name = Utils.EncodedString(dotted_name)
-        dotted_name.encoding = s.source_encoding
         return Nodes.FromImportStatNode(pos,
             module = ExprNodes.ImportNode(dotted_name_pos,
-                module_name = ExprNodes.StringNode(dotted_name_pos,
-                    value = dotted_name),
+                module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
                 name_list = import_list),
             items = items)
 
@@ -1207,7 +1222,7 @@ def p_with_statement(s):
         s.error("Only 'with gil' and 'with nogil' implemented",
                 pos = pos)
     
-def p_simple_statement(s):
+def p_simple_statement(s, first_statement = 0):
     #print "p_simple_statement:", s.sy, s.systring ###
     if s.sy == 'global':
         node = p_global_statement(s)
@@ -1226,7 +1241,7 @@ def p_simple_statement(s):
     elif s.sy in ('import', 'cimport'):
         node = p_import_statement(s)
     elif s.sy == 'from':
-        node = p_from_import_statement(s)
+        node = p_from_import_statement(s, first_statement = first_statement)
     elif s.sy == 'assert':
         node = p_assert_statement(s)
     elif s.sy == 'pass':
@@ -1235,10 +1250,10 @@ def p_simple_statement(s):
         node = p_expression_or_assignment(s)
     return node
 
-def p_simple_statement_list(s):
+def p_simple_statement_list(s, first_statement = 0):
     # Parse a series of simple statements on one line
     # separated by semicolons.
-    stat = p_simple_statement(s)
+    stat = p_simple_statement(s, first_statement = first_statement)
     if s.sy == ';':
         stats = [stat]
         while s.sy == ';':
@@ -1298,7 +1313,8 @@ def p_IF_statement(s, level, cdef_flag, visibility, api):
     s.compile_time_eval = saved_eval
     return result
 
-def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0):
+def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0,
+                first_statement = 0):
     if s.sy == 'ctypedef':
         if level not in ('module', 'module_pxd'):
             s.error("ctypedef statement not allowed here")
@@ -1361,16 +1377,18 @@ def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0):
                 elif s.sy == 'with':
                     return p_with_statement(s)
                 else:
-                    return p_simple_statement_list(s)
+                    return p_simple_statement_list(s, first_statement = first_statement)
 
 def p_statement_list(s, level,
-        cdef_flag = 0, visibility = 'private', api = 0):
+        cdef_flag = 0, visibility = 'private', api = 0, first_statement = 0):
     # Parse a series of statements separated by newlines.
     pos = s.position()
     stats = []
     while s.sy not in ('DEDENT', 'EOF'):
         stats.append(p_statement(s, level,
-            cdef_flag = cdef_flag, visibility = visibility, api = api))
+            cdef_flag = cdef_flag, visibility = visibility, api = api,
+            first_statement = first_statement))
+        first_statement = 0
     if len(stats) == 1:
         return stats[0]
     else:
@@ -2119,13 +2137,14 @@ def p_doc_string(s):
 
 def p_module(s, pxd, full_module_name):
     s.add_type_name("object")
+    s.add_type_name("Py_buffer")
     pos = s.position()
     doc = p_doc_string(s)
     if pxd:
         level = 'module_pxd'
     else:
         level = 'module'
-    body = p_statement_list(s, level)
+    body = p_statement_list(s, level, first_statement = 1)
     if s.sy != 'EOF':
         s.error("Syntax error in statement [%s,%s]" % (
             repr(s.sy), repr(s.systring)))

Cython/Compiler/PyrexTypes.py

@@ -2,6 +2,7 @@
 #   Pyrex - Types
 #
 
+from Cython import Utils
 import Naming
 
 class BaseType:
@@ -249,7 +250,7 @@ class BuiltinObjectType(PyObjectType):
         return type.is_pyobject and self.assignable_from(type)
         
     def type_test_code(self, arg):
-        return 'likely(Py%s_CheckExact(%s)) || (%s) == Py_None || (PyErr_Format(PyExc_TypeError, "Expected %s, got %%s", %s->ob_type->tp_name), 0)' % (self.name[0].upper() + self.name[1:], arg, arg, self.name, arg)
+        return 'likely(Py%s_CheckExact(%s)) || (%s) == Py_None || (PyErr_Format(PyExc_TypeError, "Expected %s, got %%s", Py_TYPE(%s)->tp_name), 0)' % (self.name[0].upper() + self.name[1:], arg, arg, self.name, arg)
 
 
 class PyExtensionType(PyObjectType):
@@ -922,23 +923,6 @@ class CEnumType(CType):
             return self.base_declaration_code(public_decl(base, dll_linkage), entity_code)
 
 
-def _escape_byte_string(s):
-    s = s.replace('\0', r'\x00')
-    try:
-        s.decode("ASCII")
-        return s
-    except UnicodeDecodeError:
-        pass
-    l = []
-    append = l.append
-    for c in s:
-        o = ord(c)
-        if o >= 128:
-            append('\\x%X' % o)
-        else:
-            append(c)
-    return ''.join(l)
-
 class CStringType:
     #  Mixin class for C string types.
 
@@ -951,7 +935,7 @@ class CStringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CUTF8StringType:
@@ -965,7 +949,7 @@ class CUTF8StringType:
 
     def literal_code(self, value):
         assert isinstance(value, str)
-        return '"%s"' % _escape_byte_string(value)
+        return '"%s"' % Utils.escape_byte_string(value)
 
 
 class CCharArrayType(CStringType, CArrayType):
@@ -998,16 +982,6 @@ class CCharPtrType(CStringType, CPtrType):
         CPtrType.__init__(self, c_char_type)
 
 
-class CUTF8CharPtrType(CUTF8StringType, CPtrType):
-    # C 'char *' type, encoded in UTF-8.
-    
-    parsetuple_format = "s"
-    pymemberdef_typecode = "T_STRING"
-    
-    def __init__(self):
-        CPtrType.__init__(self, c_char_type)
-
-
 class ErrorType(PyrexType):
     # Used to prevent propagation of error messages.
     
@@ -1074,14 +1048,17 @@ c_null_ptr_type =     CNullPtrType(c_void_type)
 c_char_array_type =   CCharArrayType(None)
 c_utf8_char_array_type =   CUTF8CharArrayType(None)
 c_char_ptr_type =     CCharPtrType()
-c_utf8_char_ptr_type =     CUTF8CharPtrType()
 c_char_ptr_ptr_type = CPtrType(c_char_ptr_type)
+c_py_ssize_t_ptr_type =  CPtrType(c_py_ssize_t_type)
 c_int_ptr_type =      CPtrType(c_int_type)
 
 c_returncode_type =   CIntType(2, 1, "T_INT", is_returncode = 1)
 
 c_anon_enum_type =    CAnonEnumType(-1, 1)
 
+# the Py_buffer type is defined in Builtin.py
+c_py_buffer_ptr_type = CPtrType(CStructOrUnionType("Py_buffer", "struct", None, 1, "Py_buffer"))
+
 error_type =    ErrorType()
 
 lowest_float_rank = 6

Cython/Compiler/Scanning.py

@@ -11,11 +11,11 @@ import stat
 import sys
 from time import time
 
-from Cython import Plex
+from Cython import Plex, Utils
 from Cython.Plex import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
-from Lexicon import string_prefixes, make_lexicon
+from Lexicon import string_prefixes, raw_prefixes, make_lexicon
 
 from Cython import Utils
 
@@ -322,6 +322,8 @@ class PyrexScanner(Scanner):
     def begin_string_action(self, text):
         if text[:1] in string_prefixes:
             text = text[1:]
+        if text[:1] in raw_prefixes:
+            text = text[1:]
         self.begin(self.string_states[text])
         self.produce('BEGIN_STRING')
     
@@ -380,8 +382,12 @@ class PyrexScanner(Scanner):
             sy, systring = self.read()
         except UnrecognizedInput:
             self.error("Unrecognized character")
-        if sy == 'IDENT' and systring in self.resword_dict:
-            sy = systring
+        if sy == 'IDENT':
+            if systring in self.resword_dict:
+                sy = systring
+            else:
+                systring = Utils.EncodedString(systring)
+                systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
         if debug_scanner:

Cython/Compiler/Symtab.py

@@ -3,6 +3,7 @@
 #
 
 import re
+from Cython import Utils
 from Errors import warning, error, InternalError
 import Options
 import Naming
@@ -15,7 +16,7 @@ from TypeSlots import \
 import ControlFlow
 import __builtin__
 
-identifier_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*$")
+possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 
 class Entry:
     # A symbol table entry in a Scope or ModuleNamespace.
@@ -64,9 +65,9 @@ class Entry:
     #                                 type is an extension type
     # as_module        None       Module scope, if a cimported module
     # is_inherited     boolean    Is an inherited attribute of an extension type
-    # #interned_cname   string     C name of interned name string
     # pystring_cname   string     C name of Python version of string literal
     # is_interned      boolean    For string const entries, value is interned
+    # is_identifier    boolean    For string const entries, value is an identifier
     # used             boolean
     # is_special       boolean    Is a special method or property accessor
     #                               of an extension type
@@ -104,8 +105,8 @@ class Entry:
     in_cinclude = 0
     as_module = None
     is_inherited = 0
-    #interned_cname = None
     pystring_cname = None
+    is_identifier = 0
     is_interned = 0
     used = 0
     is_special = 0
@@ -188,6 +189,7 @@ class Scope:
         self.cname_to_entry = {}
         self.pow_function_used = 0
         self.string_to_entry = {}
+        self.identifier_to_entry = {}
         self.num_to_entry = {}
         self.obj_to_entry = {}
         self.pystring_entries = []
@@ -204,10 +206,10 @@ class Scope:
         
     def __str__(self):
         return "<%s %s>" % (self.__class__.__name__, self.qualified_name)
-    
-    def intern(self, name):
-        return self.global_scope().intern(name)
-    
+
+    def intern_identifier(self, name):
+        return self.global_scope().intern_identifier(name)
+
     def qualifying_scope(self):
         return self.parent_scope
     
@@ -448,31 +450,36 @@ class Scope:
         self.const_entries.append(entry)
         return entry
 
-    def get_string_const(self, value):
+    def get_string_const(self, value, identifier = False):
         # Get entry for string constant. Returns an existing
         # one if possible, otherwise creates a new one.
         genv = self.global_scope()
-        entry = genv.string_to_entry.get(value)
+        if identifier:
+            string_map = genv.identifier_to_entry
+        else:
+            string_map = genv.string_to_entry
+        entry = string_map.get(value)
         if not entry:
             entry = self.add_string_const(value)
-            genv.string_to_entry[value] = entry
+            entry.is_identifier = identifier
+            string_map[value] = entry
         return entry
 
-    def add_py_string(self, entry):
+    def add_py_string(self, entry, identifier = None):
         # If not already done, allocate a C name for a Python version of
         # a string literal, and add it to the list of Python strings to
         # be created at module init time. If the string resembles a
         # Python identifier, it will be interned.
-        if not entry.pystring_cname:
-            value = entry.init
-            if not entry.type.is_unicode and identifier_pattern.match(value):
-                entry.pystring_cname = self.intern(value)
-                entry.is_interned = 1
-            else:
-                entry.pystring_cname = entry.cname + "p"
-                self.pystring_entries.append(entry)
-                self.global_scope().all_pystring_entries.append(entry)
-                
+        if entry.pystring_cname:
+            return
+        value = entry.init
+        entry.pystring_cname = entry.cname + "p"
+        self.pystring_entries.append(entry)
+        self.global_scope().all_pystring_entries.append(entry)
+        if identifier or (identifier is None and possible_identifier(value)):
+            entry.is_interned = 1
+            self.global_scope().new_interned_string_entries.append(entry)
+
     def add_py_num(self, value):
         # Add an entry for an int constant.
         cname = "%s%s" % (Naming.interned_num_prefix, value)
@@ -605,11 +612,14 @@ class BuiltinScope(Scope):
             utility_code = None):
         # If python_equiv == "*", the Python equivalent has the same name
         # as the entry, otherwise it has the name specified by python_equiv.
+        name = Utils.EncodedString(name)
         entry = self.declare_cfunction(name, type, None, cname)
         entry.utility_code = utility_code
         if python_equiv:
             if python_equiv == "*":
                 python_equiv = name
+            else:
+                python_equiv = Utils.EncodedString(python_equiv)
             var_entry = Entry(python_equiv, python_equiv, py_object_type)
             var_entry.is_variable = 1
             var_entry.is_builtin = 1
@@ -617,6 +627,7 @@ class BuiltinScope(Scope):
         return entry
         
     def declare_builtin_type(self, name, cname):
+        name = Utils.EncodedString(name)
         type = PyrexTypes.BuiltinObjectType(name, cname)
         type.set_scope(CClassScope(name, outer_scope=None, visibility='extern'))
         self.type_names[name] = 1
@@ -671,14 +682,14 @@ class ModuleScope(Scope):
     # python_include_files [string]           Standard  Python headers to be included
     # include_files        [string]           Other C headers to be included
     # string_to_entry      {string : Entry}   Map string const to entry
+    # identifier_to_entry  {string : Entry}   Map identifier string const to entry
     # context              Context
     # parent_module        Scope              Parent in the import namespace
     # module_entries       {string : Entry}   For cimport statements
     # type_names           {string : 1}       Set of type names (used during parsing)
     # pxd_file_loaded      boolean            Corresponding .pxd file has been processed
     # cimported_modules    [ModuleScope]      Modules imported with cimport
-    # intern_map           {string : string}  Mapping from Python names to interned strs
-    # interned_names       [string]           Interned names pending generation of declarations
+    # new_interned_string_entries [Entry]     New interned strings waiting to be declared
     # interned_nums        [int/long]         Interned numeric constants
     # all_pystring_entries [Entry]            Python string consts from all scopes
     # types_imported       {PyrexType : 1}    Set of types for which import code generated
@@ -705,8 +716,7 @@ class ModuleScope(Scope):
         self.type_names = dict(outer_scope.type_names)
         self.pxd_file_loaded = 0
         self.cimported_modules = []
-        self.intern_map = {}
-        self.interned_names = []
+        self.new_interned_string_entries = []
         self.interned_nums = []
         self.interned_objs = []
         self.all_pystring_entries = []
@@ -743,15 +753,11 @@ class ModuleScope(Scope):
         else:
             entry.is_builtin = 1
         return entry
-    
-    def intern(self, name):
-        intern_map = self.intern_map
-        cname = intern_map.get(name)
-        if not cname:
-            cname = Naming.interned_prefix + name
-            intern_map[name] = cname
-            self.interned_names.append(name)
-        return cname
+
+    def intern_identifier(self, name):
+        string_entry = self.get_string_const(name, identifier = True)
+        self.add_py_string(string_entry, identifier = 1)
+        return string_entry.pystring_cname
 
     def find_module(self, module_name, pos):
         # Find a module in the import namespace, interpreting
@@ -832,8 +838,6 @@ class ModuleScope(Scope):
                     "Non-cdef global variable is not a generic Python object")
             entry.is_pyglobal = 1
             entry.namespace_cname = self.module_cname
-            #if Options.intern_names:
-            #	entry.interned_cname = self.intern(name)
         else:
             entry.is_cglobal = 1
             self.var_entries.append(entry)
@@ -1151,8 +1155,6 @@ class PyClassScope(ClassScope):
             cname, visibility, is_cdef)
         entry.is_pyglobal = 1
         entry.namespace_cname = self.class_obj_cname
-        #if Options.intern_names:
-        #	entry.interned_cname = self.intern(name)
         return entry
 
     def allocate_temp(self, type):
@@ -1250,10 +1252,9 @@ class CClassScope(ClassScope):
                                   # I keep it in for now. is_member should be enough
                                   # later on
             entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
-            if Options.intern_names:
-                entry.interned_cname = self.intern(name)
+            entry.interned_cname = self.intern_identifier(name)
             return entry
-            
+
 
     def declare_pyfunction(self, name, pos):
         # Add an entry for a method.
@@ -1262,7 +1263,7 @@ class CClassScope(ClassScope):
         if name == "__new__":
             warning(pos, "__new__ method of extension type will change semantics "
                 "in a future version of Pyrex and Cython. Use __cinit__ instead.")
-            name = "__cinit__"
+            name = Utils.EncodedString("__cinit__")
         entry = self.declare_var(name, py_object_type, pos)
         special_sig = get_special_method_signature(name)
         if special_sig:
@@ -1279,7 +1280,7 @@ class CClassScope(ClassScope):
     
     def lookup_here(self, name):
         if name == "__new__":
-            name = "__cinit__"
+            name = Utils.EncodedString("__cinit__")
         return ClassScope.lookup_here(self, name)
     
     def declare_cfunction(self, name, type, pos,
@@ -1401,13 +1402,16 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/
 static PyObject* __Pyx_Method_ClassMethod(PyObject *method) {
     /* It appears that PyMethodDescr_Type is not anywhere exposed in the Python/C API */
     /* if (!PyObject_TypeCheck(method, &PyMethodDescr_Type)) { */ 
-    if (strcmp(method->ob_type->tp_name, "method_descriptor") == 0) { /* cdef classes */
+    if (strcmp(Py_TYPE(method)->tp_name, "method_descriptor") == 0) { /* cdef classes */
         PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
         return PyDescr_NewClassMethod(descr->d_type, descr->d_method);
     }
     else if (PyMethod_Check(method)) {                                /* python classes */
         return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
     }
+    else if (PyCFunction_Check(method)) {
+        return PyClassMethod_New(method);
+    }
     PyErr_Format(PyExc_TypeError, "Class-level classmethod() can only be called on a method_descriptor or instance method.");
     return NULL;
 }

Cython/Compiler/TypeSlots.py

@@ -3,6 +3,7 @@
 #           and associated know-how.
 #
 
+from Cython import Utils
 import Naming
 import PyrexTypes
 import sys
@@ -25,12 +26,14 @@ class Signature:
     #    'p'  void *
     #    'P'  void **
     #    'i'  int
+    #    'b'  bint
     #    'I'  int *
     #    'l'  long
     #    'Z'  Py_ssize_t
     #    's'  char *
     #    'S'  char **
     #    'r'  int used only to signal exception
+    #    'B'  Py_buffer *
     #    '-'  dummy 'self' argument (not used)
     #    '*'  rest of args passed as generic Python
     #           arg tuple and kw dict (must be last
@@ -49,6 +52,7 @@ class Signature:
         's': PyrexTypes.c_char_ptr_type,
         'S': PyrexTypes.c_char_ptr_ptr_type,
         'r': PyrexTypes.c_returncode_type,
+        'B': PyrexTypes.c_py_buffer_ptr_type,
         # 'T', '-' and '*' are handled otherwise
         # and are not looked up in here
     }
@@ -121,11 +125,15 @@ class SlotDescriptor:
     #  slot_name    string           Member name of the slot in the type object
     #  is_initialised_dynamically    Is initialised by code in the module init function
     #  flag                          Py_TPFLAGS_XXX value indicating presence of slot
-    
-    def __init__(self, slot_name, dynamic = 0, flag = None):
+    #  py3k                          Indicates presence of slot in Python 3
+    #  py2                           Indicates presence of slot in Python 2
+
+    def __init__(self, slot_name, dynamic = 0, flag = None, py3k = True, py2 = True):
         self.slot_name = slot_name
         self.is_initialised_dynamically = dynamic
         self.flag = flag
+        self.py3k = py3k
+        self.py2  = py2
 
     def generate(self, scope, code):
         if self.is_initialised_dynamically:
@@ -133,11 +141,19 @@ class SlotDescriptor:
         else:
             value = self.slot_code(scope)
         flag = self.flag
+        py3k = self.py3k
+        py2  = self.py2
+        if not py3k:
+            code.putln("#if PY_MAJOR_VERSION < 3")
+        elif not py2:
+            code.putln("#if PY_MAJOR_VERSION >= 3")
         if flag:
-            code.putln("#if Py_TPFLAGS_DEFAULT & %s" % flag)
+            code.putln("#if (PY_MAJOR_VERSION >= 3) || (Py_TPFLAGS_DEFAULT & %s)" % flag)
         code.putln("%s, /*%s*/" % (value, self.slot_name))
         if flag:
             code.putln("#endif")
+        if not py3k or not py2:
+            code.putln("#endif")
 
     # Some C implementations have trouble statically 
     # initialising a global with a pointer to an extern 
@@ -183,8 +199,8 @@ class MethodSlot(SlotDescriptor):
     #  method_name  string           The __xxx__ name of the method
     #  default      string or None   Default value of the slot
     
-    def __init__(self, signature, slot_name, method_name, default = None, flag = None):
-        SlotDescriptor.__init__(self, slot_name, flag = flag)
+    def __init__(self, signature, slot_name, method_name, default = None, flag = None, py3k=True, py2=True):
+        SlotDescriptor.__init__(self, slot_name, flag = flag, py3k = py3k, py2=py2)
         self.signature = signature
         self.slot_name = slot_name
         self.method_name = method_name
@@ -205,8 +221,8 @@ class InternalMethodSlot(SlotDescriptor):
     #
     #  slot_name    string           Member name of the slot in the type object
 
-    def __init__(self, slot_name):
-        SlotDescriptor.__init__(self, slot_name)
+    def __init__(self, slot_name, py2 = True):
+        SlotDescriptor.__init__(self, slot_name, py2 = py2)
 
     def slot_code(self, scope):
         return scope.mangle_internal(self.slot_name)
@@ -264,8 +280,8 @@ class SyntheticSlot(InternalMethodSlot):
     #  alternative default value will be placed in the type
     #  slot.
     
-    def __init__(self, slot_name, user_methods, default_value):
-        InternalMethodSlot.__init__(self, slot_name)
+    def __init__(self, slot_name, user_methods, default_value, py2 = True):
+        InternalMethodSlot.__init__(self, slot_name, py2 = py2)
         self.user_methods = user_methods
         self.default_value = default_value
     
@@ -291,7 +307,11 @@ class DocStringSlot(SlotDescriptor):
     
     def slot_code(self, scope):
         if scope.doc is not None:
-            return '"%s"' % scope.doc
+            if scope.doc.is_unicode:
+                doc = scope.doc.utf8encode()
+            else:
+                doc = scope.doc.byteencode()
+            return '"%s"' % Utils.escape_byte_string(doc)
         else:
             return "0"
 
@@ -492,6 +512,10 @@ initproc = Signature("T*", 'r')            # typedef int (*initproc)(PyObject *,
                                            # typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *);
                                            # typedef PyObject *(*allocfunc)(struct _typeobject *, int);
 
+getbufferproc = Signature("TBi", "r")      # typedef int (*getbufferproc)(PyObject *, Py_buffer *, int);
+releasebufferproc = Signature("TB", "v")   # typedef void (*releasebufferproc)(PyObject *, Py_buffer *);
+
+
 #------------------------------------------------------------------------------------------
 #
 #  Signatures for accessor methods of properties.
@@ -515,7 +539,7 @@ PyNumberMethods = (
     MethodSlot(binaryfunc, "nb_add", "__add__"),
     MethodSlot(binaryfunc, "nb_subtract", "__sub__"),
     MethodSlot(binaryfunc, "nb_multiply", "__mul__"),
-    MethodSlot(binaryfunc, "nb_divide", "__div__"),
+    MethodSlot(binaryfunc, "nb_divide", "__div__", py3k = False),
     MethodSlot(binaryfunc, "nb_remainder", "__mod__"),
     MethodSlot(binaryfunc, "nb_divmod", "__divmod__"),
     MethodSlot(ternaryfunc, "nb_power", "__pow__"),
@@ -540,7 +564,7 @@ PyNumberMethods = (
     MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"),
     MethodSlot(ibinaryfunc, "nb_inplace_subtract", "__isub__"),
     MethodSlot(ibinaryfunc, "nb_inplace_multiply", "__imul__"),
-    MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__"),
+    MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", py3k = False),
     MethodSlot(ibinaryfunc, "nb_inplace_remainder", "__imod__"),
     MethodSlot(ternaryfunc, "nb_inplace_power", "__ipow__"), # NOT iternaryfunc!!!
     MethodSlot(ibinaryfunc, "nb_inplace_lshift", "__ilshift__"),
@@ -580,10 +604,13 @@ PyMappingMethods = (
 )
 
 PyBufferProcs = (
-    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__"),
-    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__"),
-    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__"),
-    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__"),
+    MethodSlot(getreadbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3k = False),
+    MethodSlot(getwritebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3k = False),
+    MethodSlot(getsegcountproc, "bf_getsegcount", "__getsegcount__", py3k = False),
+    MethodSlot(getcharbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3k = False),
+
+    MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", flag = "Py_TPFLAGS_HAVE_NEWBUFFER"),
+    MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", flag = "Py_TPFLAGS_HAVE_NEWBUFFER"),
 )
 
 #------------------------------------------------------------------------------------------

Cython/Utils.py

@@ -91,3 +91,20 @@ class EncodedString(unicode):
 #    def __eq__(self, other):
 #        return unicode.__eq__(self, other) and \
 #            getattr(other, 'encoding', '') == self.encoding
+
+def escape_byte_string(s):
+    s = s.replace('\0', r'\x00')
+    try:
+        s.decode("ASCII")
+        return s
+    except UnicodeDecodeError:
+        pass
+    l = []
+    append = l.append
+    for c in s:
+        o = ord(c)
+        if o >= 128:
+            append('\\x%X' % o)
+        else:
+            append(c)
+    return ''.join(l)

Makefile

@@ -17,3 +17,7 @@ testclean:
 
 test:	testclean
 	${PYTHON} runtests.py
+
+test3:	testclean
+	${PYTHON} runtests.py --no-cleanup
+	python3.0 runtests.py --no-cython

runtests.py

@@ -2,11 +2,7 @@
 
 import os, sys, re, shutil, unittest, doctest
 
-from Cython.Compiler.Version import version
-from Cython.Compiler.Main import \
-    CompilationOptions, \
-    default_options as pyrex_default_options, \
-    compile as cython_compile
+WITH_CYTHON = True
 
 from distutils.dist import Distribution
 from distutils.core import Extension
@@ -37,17 +33,21 @@ class ErrorWriter(object):
         return errors
 
 class TestBuilder(object):
-    def __init__(self, rootdir, workdir, selectors, annotate):
+    def __init__(self, rootdir, workdir, selectors, annotate, cleanup_workdir):
         self.rootdir = rootdir
         self.workdir = workdir
         self.selectors = selectors
         self.annotate = annotate
+        self.cleanup_workdir = cleanup_workdir
 
     def build_suite(self):
         suite = unittest.TestSuite()
         filenames = os.listdir(self.rootdir)
         filenames.sort()
         for filename in filenames:
+            if not WITH_CYTHON and filename == "errors":
+                # we won't get any errors without running Cython
+                continue
             path = os.path.join(self.rootdir, filename)
             if os.path.isdir(path) and filename in TEST_DIRS:
                 suite.addTest(
@@ -55,6 +55,12 @@ class TestBuilder(object):
         return suite
 
     def handle_directory(self, path, context):
+        workdir = os.path.join(self.workdir, context)
+        if not os.path.exists(workdir):
+            os.makedirs(workdir)
+        if workdir not in sys.path:
+            sys.path.insert(0, workdir)
+
         expect_errors = (context == 'errors')
         suite = unittest.TestSuite()
         filenames = os.listdir(path)
@@ -69,29 +75,38 @@ class TestBuilder(object):
                 continue
             if context in TEST_RUN_DIRS:
                 test = CythonRunTestCase(
-                    path, self.workdir, module, self.annotate)
+                    path, workdir, module,
+                    annotate=self.annotate,
+                    cleanup_workdir=self.cleanup_workdir)
             else:
                 test = CythonCompileTestCase(
-                    path, self.workdir, module, expect_errors, self.annotate)
+                    path, workdir, module,
+                    expect_errors=expect_errors,
+                    annotate=self.annotate,
+                    cleanup_workdir=self.cleanup_workdir)
             suite.addTest(test)
         return suite
 
 class CythonCompileTestCase(unittest.TestCase):
     def __init__(self, directory, workdir, module,
-                 expect_errors=False, annotate=False):
+                 expect_errors=False, annotate=False, cleanup_workdir=True):
         self.directory = directory
         self.workdir = workdir
         self.module = module
         self.expect_errors = expect_errors
         self.annotate = annotate
+        self.cleanup_workdir = cleanup_workdir
         unittest.TestCase.__init__(self)
 
     def shortDescription(self):
         return "compiling " + self.module
 
     def tearDown(self):
+        cleanup_c_files = WITH_CYTHON and self.cleanup_workdir
         if os.path.exists(self.workdir):
             for rmfile in os.listdir(self.workdir):
+                if not cleanup_c_files and rmfile[-2:] in (".c", ".h"):
+                    continue
                 if self.annotate and rmfile.endswith(".html"):
                     continue
                 try:
@@ -171,13 +186,14 @@ class CythonCompileTestCase(unittest.TestCase):
                 directory, module, workdir)
             directory = workdir
 
-        old_stderr = sys.stderr
-        try:
-            sys.stderr = ErrorWriter()
-            self.run_cython(directory, module, workdir, incdir, annotate)
-            errors = sys.stderr.geterrors()
-        finally:
-            sys.stderr = old_stderr
+        if WITH_CYTHON:
+            old_stderr = sys.stderr
+            try:
+                sys.stderr = ErrorWriter()
+                self.run_cython(directory, module, workdir, incdir, annotate)
+                errors = sys.stderr.geterrors()
+            finally:
+                sys.stderr = old_stderr
 
         if errors or expected_errors:
             for expected, error in zip(expected_errors, errors):
@@ -198,11 +214,11 @@ class CythonRunTestCase(CythonCompileTestCase):
     def run(self, result=None):
         if result is None:
             result = self.defaultTestResult()
+        result.startTest(self)
         try:
             self.runTest()
             doctest.DocTestSuite(self.module).run(result)
         except Exception:
-            result.startTest(self)
             result.addError(self, sys.exc_info())
             result.stopTest(self)
         try:
@@ -211,49 +227,68 @@ class CythonRunTestCase(CythonCompileTestCase):
             pass
 
 if __name__ == '__main__':
-    # RUN ALL TESTS!
-    ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
-    WORKDIR = os.path.join(os.getcwd(), 'BUILD')
-    if os.path.exists(WORKDIR):
-        shutil.rmtree(WORKDIR, ignore_errors=True)
-    os.makedirs(WORKDIR)
-
-    if not sys.path or sys.path[0] != WORKDIR:
-        sys.path.insert(0, WORKDIR)
-
-    print "Running tests against Cython %s" % version
-    print "Python", sys.version
-    print
-
-    try:
-        sys.argv.remove("-C")
-    except ValueError:
-        coverage = None
-    else:
+    from optparse import OptionParser
+    parser = OptionParser()
+    parser.add_option("--no-cleanup", dest="cleanup_workdir",
+                      action="store_false", default=True,
+                      help="do not delete the generated C files (allows passing --no-cython on next run)")
+    parser.add_option("--no-cython", dest="with_cython",
+                      action="store_false", default=True,
+                      help="do not run the Cython compiler, only the C compiler")
+    parser.add_option("-C", "--coverage", dest="coverage",
+                      action="store_true", default=False,
+                      help="collect source coverage data for the Compiler")
+    parser.add_option("-A", "--annotate", dest="annotate_source",
+                      action="store_true", default=False,
+                      help="generate annotated HTML versions of the test source files")
+    parser.add_option("-v", "--verbose", dest="verbosity",
+                      action="count", default=0,
+                      help="display test progress, pass twice to print test names")
+
+    options, cmd_args = parser.parse_args()
+
+    if options.coverage:
         import coverage
         coverage.erase()
+        coverage.start()
+
+    WITH_CYTHON = options.with_cython
+
+    if WITH_CYTHON:
+        from Cython.Compiler.Main import \
+            CompilationOptions, \
+            default_options as pyrex_default_options, \
+            compile as cython_compile
 
-    try:
-        sys.argv.remove("-a")
-    except ValueError:
-        annotate_source = False
+    # RUN ALL TESTS!
+    ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')
+    WORKDIR = os.path.join(os.getcwd(), 'BUILD')
+    if WITH_CYTHON:
+        if os.path.exists(WORKDIR):
+            shutil.rmtree(WORKDIR, ignore_errors=True)
+    if not os.path.exists(WORKDIR):
+        os.makedirs(WORKDIR)
+
+    if WITH_CYTHON:
+        from Cython.Compiler.Version import version
+        print("Running tests against Cython %s" % version)
     else:
-        annotate_source = True
+        print("Running tests without Cython.")
+    print("Python %s" % sys.version)
+    print("")
 
     import re
-    selectors = [ re.compile(r, re.I).search for r in sys.argv[1:] ]
+    selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
     if not selectors:
         selectors = [ lambda x:True ]
 
-    tests = TestBuilder(ROOTDIR, WORKDIR, selectors, annotate_source)
+    tests = TestBuilder(ROOTDIR, WORKDIR, selectors,
+                        options.annotate_source, options.cleanup_workdir)
     test_suite = tests.build_suite()
 
-    if coverage is not None:
-        coverage.start()
-
-    unittest.TextTestRunner(verbosity=2).run(test_suite)
+    unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 
-    if coverage is not None:
+    if options.coverage:
         coverage.stop()
         ignored_modules = ('Options', 'Version', 'DebugFlags')
         modules = [ module for name, module in sys.modules.items()

tests/broken/ass2cglobal.pyx

-cdef int i
-cdef x
-
-def f(a):
-    global i, x
-    i = 42
-    x = a

tests/broken/cstringmeth.pyx

-x = "foo".join(y)
-

tests/broken/filenames.pxi

-spam = "ftang"

tests/broken/inhcmethcall.pyx

-cdef class Parrot:
-
-  cdef void describe(self):
-    pass
-
-
-cdef class Norwegian(Parrot):
-
-  cdef void describe(self):
-    Parrot.describe(self)

tests/compile/classmethargdefault.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Swallow()
     >>> s.spam(1)
     1 42 'grail' True

tests/compile/dotted_cimport.pyx

+cimport dotted_cimport_submodule.a
+import dotted_cimport_submodule.b

tests/broken/eqcmp.pyx → tests/compile/eqcmp.pyx

@@ -4,7 +4,6 @@ cdef void foo():
     cdef char *ptr1, *ptr2
     cdef int *ptr3
     bool = int1 == int2
-    bool = int1 <> int2
     bool = int1 != int2
     bool = float1 == float2
     bool = ptr1 == ptr2

tests/broken/extcoerce.pyx → tests/compile/extcoerce.pyx

@@ -2,12 +2,6 @@ cdef class Grail:
 
     def __add__(int x, float y):
         pass
-    
-    def __getslice__(self, i, j):
-        pass
-
-    def __setslice__(self, Py_ssize_t i, float j, x):
-        pass
 
 cdef class Swallow:
     pass

tests/compile/extinheritdel.pyx

@@ -5,9 +5,6 @@ cdef class Norwegian(Parrot):
     
     def __delitem__(self, i):
         pass
-    
-    def __delslice__(self, i, j):
-        pass
 
     def __delattr__(self, n):
         pass

tests/compile/extinheritset.pyx

@@ -5,9 +5,6 @@ cdef class Norwegian(Parrot):
     
     def __setitem__(self, i, x):
         pass
-    
-    def __setslice__(self, i, j, x):
-        pass
 
     def __setattr__(self, n, x):
         pass

tests/compile/food.h

+struct Tomato {
	PyObject_HEAD};
struct Bicycle{
	PyObject_HEAD};
\ No newline at end of file

tests/compile/pinard4.pyx

-__doc__ = """
+__doc__ = u"""
     >>> fiches_CP
     []
 """

tests/broken/r_pernici1.pyx → tests/compile/r_pernici1.pyx

+__doc__ = u"""
+>>> main()
+3.14159265358979323846
+3.14159265358979323846
+3.14159265358979323846
+"""
+
 cdef extern from "math.h":
   double M_PI
 

tests/errors/cdefoptargs.pyx

+def call5():
+    b(1,2,3,4,5)
+
+cdef b(a, b, c=1, d=2):
+    pass
+
+_ERRORS = """
+2:5:Call with wrong number of arguments (expected at most 4, got 5)
+"""

tests/run/__getattribute__.pyx

-__doc__ = """
+__doc__ = u"""
 __getattribute__ and __getattr__ special methods for a single class.
 
     >>> a = just_getattribute()
@@ -29,7 +29,7 @@ __getattribute__ and __getattr__ special methods for a single class.
 
 cdef class just_getattribute:
     def __getattribute__(self,n):
-        if n == 'bar':
+        if n == u'bar':
             return n
         else:
             raise AttributeError
@@ -39,7 +39,7 @@ cdef class just_getattr:
     def __init__(self):
         self.foo = 10
     def __getattr__(self,n):
-        if n == 'bar':
+        if n == u'bar':
             return n
         else:
             raise AttributeError
@@ -49,12 +49,12 @@ cdef class both:
     def __init__(self):
         self.foo = 10
     def __getattribute__(self,n):
-        if n == 'foo':
+        if n == u'foo':
             return self.foo
         else:
             raise AttributeError
     def __getattr__(self,n):
-        if n == 'bar':
+        if n == u'bar':
             return n
         else:
             raise AttributeError

tests/run/__getattribute_subclasses__.pyx

-__doc__ = """
+__doc__ = u"""
 __getattribute__ and __getattr__ special methods and subclasses. 
 
 getattr does not override members. 
     >>> a = getattr_boring()
     >>> a.boring_member
     10
-    >>> a.resolved_by
-    'getattr_boring'
+    >>> print(a.resolved_by)
+    getattr_boring
 
 getattribute does.
     >>> a = getattribute_boring()
     >>> a.boring_member
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    'getattribute_boring'
+    >>> print(a.resolved_by)
+    getattribute_boring
 
 Is inherited.
     >>> a = boring_boring_getattribute()
@@ -24,8 +24,8 @@ Is inherited.
     >>> a.boring_boring_getattribute_member
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    '_getattribute'
+    >>> print(a.resolved_by)
+    _getattribute
 
 __getattribute__ is always tried first, then __getattr__, regardless of where
 in the inheritance hiarchy they came from.
@@ -33,8 +33,8 @@ in the inheritance hiarchy they came from.
     >>> a.foo
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    'getattribute_boring_boring_getattr'
+    >>> print(a.resolved_by)
+    getattribute_boring_boring_getattr
     >>> a.getattribute_boring_boring_getattr
     True
     >>> a._getattr
@@ -44,8 +44,8 @@ in the inheritance hiarchy they came from.
     >>> a.foo
     Traceback (most recent call last):
     AttributeError
-    >>> a.resolved_by
-    '_getattribute'
+    >>> print(a.resolved_by)
+    _getattribute
     >>> a.getattr_boring_boring_getattribute
     True
     >>> a._getattribute
@@ -60,36 +60,36 @@ cdef class boring:
 
 cdef class getattr_boring(boring):
     def __getattr__(self,n):
-        if n == 'resolved_by':
-            return 'getattr_boring'
-        elif n == 'getattr_boring':
+        if n == u'resolved_by':
+            return u'getattr_boring'
+        elif n == u'getattr_boring':
             return True
         else:
             raise AttributeError
 
 cdef class getattribute_boring(boring):
     def __getattribute__(self,n):
-        if n == 'resolved_by':
-            return 'getattribute_boring'
-        elif n == 'getattribute_boring':
+        if n == u'resolved_by':
+            return u'getattribute_boring'
+        elif n == u'getattribute_boring':
             return True
         else:
             raise AttributeError
 
 cdef class _getattr:
     def __getattr__(self,n):
-        if n == 'resolved_by':
-            return '_getattr'
-        elif n == '_getattr':
+        if n == u'resolved_by':
+            return u'_getattr'
+        elif n == u'_getattr':
             return True
         else:
             raise AttributeError
 
 cdef class _getattribute(boring):
     def __getattribute__(self,n):
-        if n == 'resolved_by':
-            return '_getattribute'
-        elif n == '_getattribute':
+        if n == u'resolved_by':
+            return u'_getattribute'
+        elif n == u'_getattribute':
             return True
         else:
             raise AttributeError
@@ -108,19 +108,18 @@ cdef class boring_boring_getattr(boring_getattr):
 
 cdef class getattribute_boring_boring_getattr(boring_boring_getattr):
     def __getattribute__(self,n):
-        if n == 'resolved_by':
-            return 'getattribute_boring_boring_getattr'
-        elif n == 'getattribute_boring_boring_getattr':
+        if n == u'resolved_by':
+            return u'getattribute_boring_boring_getattr'
+        elif n == u'getattribute_boring_boring_getattr':
             return True
         else:
             raise AttributeError
 
 cdef class getattr_boring_boring_getattribute(boring_boring_getattribute):
     def __getattr__(self,n):
-        if n == 'resolved_by':
-            return 'getattr_boring_boring_getattribute'
-        elif n == 'getattr_boring_boring_getattribute':
+        if n == u'resolved_by':
+            return u'getattr_boring_boring_getattribute'
+        elif n == u'getattr_boring_boring_getattribute':
             return True
         else:
             raise AttributeError
-

tests/run/addloop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> x = 1
     >>> for i in range(10):
     ...     x = x + i

tests/run/addop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     (30, 22)
 """

tests/run/addressof.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(5)
     5
 """

tests/run/altet2.pyx

-__doc__ = """
+__doc__ = u"""
   >>> iter(C())
   Traceback (most recent call last):
   TypeError: iter() returned non-iterator of type 'NoneType'

tests/broken/anonymousenum.pyx → tests/run/anonymousenum.pyx

+__doc__ = u"""
+>>> p
+42
+"""
+
 cdef enum:
     spam = 42
     grail = 17
@@ -5,3 +10,4 @@ cdef enum:
 cdef int i
 i = spam
 
+p = i

tests/run/append.pyx

-__doc__ = """
+__doc__ = u"""
 >>> test_append([])
 None
 None
@@ -23,9 +23,9 @@ None
 
 class A:
     def append(self, x):
-        print "appending"
+        print u"appending"
         return x
-        
+
 class B(list):
     def append(self, *args):
         for arg in args:
@@ -38,6 +38,6 @@ def test_append(L):
     try:
         print L.append(5,6)
     except TypeError:
-        print "got error"
+        print u"got error"
     return L
 

tests/run/ass2cglobal.pyx

+__doc__ = u"""
+>>> what()
+0 5
+>>> f(5)
+>>> what()
+42 5
+>>> f(6)
+>>> what()
+42 6
+>>> f("spam")
+>>> what()
+42 spam
+"""
+
+cdef int i = 0
+cdef x = 5
+
+def f(a):
+    global i, x
+    i = 42
+    x = a
+
+def what():
+    print i,x

tests/run/ass2global.pyx

-__doc__ = """
+__doc__ = u"""
     >>> getg()
     5
     >>> f(42)

tests/run/ass2local.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     42
 """

tests/run/assert.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1, 2, 1)
     >>> f(0, 2, 1)
     Traceback (most recent call last):

tests/run/attr.pyx

-__doc__ = """
-    >>> class Test:
+__doc__ = u"""
+    >>> class Test(object):
     ...     def __init__(self, i):
     ...         self.i = i
     >>> b = Test(1)
@@ -9,50 +9,50 @@ __doc__ = """
     >>> b.spam.eggs.spam.eggs = Test(5)
 
     >>> a = f(b)
-    >>> print a.i
+    >>> a.i
     2
-    >>> print b.i
+    >>> b.i
     1
-    >>> print a.spam.i
+    >>> a.spam.i
     1
-    >>> print b.spam.i
+    >>> b.spam.i
     2
-    >>> print a.spam.eggs.i
+    >>> a.spam.eggs.i
     Traceback (most recent call last):
-    AttributeError: Test instance has no attribute 'eggs'
-    >>> print b.spam.eggs.i
+    AttributeError: 'Test' object has no attribute 'eggs'
+    >>> b.spam.eggs.i
     3
-    >>> print a.spam.spam.i
+    >>> a.spam.spam.i
     2
-    >>> print b.spam.spam.i
+    >>> b.spam.spam.i
     1
-    >>> print a.spam.eggs.spam.i
+    >>> a.spam.eggs.spam.i
     Traceback (most recent call last):
-    AttributeError: Test instance has no attribute 'eggs'
-    >>> print b.spam.eggs.spam.i
+    AttributeError: 'Test' object has no attribute 'eggs'
+    >>> b.spam.eggs.spam.i
     4
 
     >>> a = g(b)
-    >>> print a.i
+    >>> a.i
     3
-    >>> print b.i
+    >>> b.i
     1
-    >>> print a.spam.i
+    >>> a.spam.i
     4
-    >>> print b.spam.i
+    >>> b.spam.i
     2
-    >>> print a.spam.eggs.i
+    >>> a.spam.eggs.i
     1
-    >>> print b.spam.eggs.i
+    >>> b.spam.eggs.i
     3
-    >>> print a.spam.spam.i
+    >>> a.spam.spam.i
     Traceback (most recent call last):
-    AttributeError: Test instance has no attribute 'spam'
-    >>> print b.spam.spam.i
+    AttributeError: 'Test' object has no attribute 'spam'
+    >>> b.spam.spam.i
     1
-    >>> print a.spam.eggs.spam.i
+    >>> a.spam.eggs.spam.i
     2
-    >>> print b.spam.eggs.spam.i
+    >>> b.spam.eggs.spam.i
     4
 """
 

tests/run/baas3.pyx

-__doc__ = """
+__doc__ = u"""
     >>> m = MyClass()
     >>> m is foo(m)
     True

tests/run/backquote.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(20)
     '20'
     >>> f('test')

tests/run/behnel1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> viking(5)
     5
 """

tests/run/behnel2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> y
     1
     >>> y and {}

tests/run/behnel3.pyx

-__doc__ = """
+__doc__ = u"""
     >>> y
     >>> y or {}
     {}

tests/run/bishop1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> m = fmatrix()
     >>> m[1] = True
     >>> m.getfoo()

tests/run/bishop2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f = foo()
     >>> 'a' in f
     True

tests/run/boolop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> foo(True, False, 23, 'test', 1)
     (0, 1, False, False)
 """

tests/run/buffer.pyx

+__doc__ = u"""
+>>> b1 = TestBuffer()
+>>> b2 = TestBufferRelease()
+"""
+
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ += u"""
+>>> ms = memoryview(s)
+>>> ms.tobytes()
+bytearray(b'abcdefg')
+
+>>> m1 = memoryview(b1)
+>>> m1.tobytes()
+locking!
+bytearray(b'abcdefg')
+
+>>> m2 = memoryview(b2)
+>>> m2.tobytes()
+locking!
+unlocking!
+bytearray(b'abcdefg')
+
+>>> del m1
+>>> del m2
+releasing!
+"""
+
+s = "abcdefg"
+
+cdef class TestBuffer:
+    def __getbuffer__(self, Py_buffer* buffer, int flags):
+        if buffer is NULL:
+            print u"locking!"
+            return
+        buffer.buf = <char*>s
+        buffer.len = len(s)
+        buffer.readonly = 0
+        buffer.format = "B"
+        buffer.ndim = 0
+        buffer.shape = NULL
+        buffer.strides = NULL
+        buffer.suboffsets = NULL
+        buffer.itemsize = 1
+        buffer.internal = NULL
+
+cdef class TestBufferRelease(TestBuffer):
+    def __releasebuffer__(self, Py_buffer* buffer):
+        if buffer is NULL:
+            print u"unlocking!"
+        else:
+            print u"releasing!"

tests/run/cdefoptargs.pyx

+__doc__ = u"""
+    >>> call2()
+    >>> call3()
+    >>> call4()
+"""
+
+# the calls:
+
+def call2():
+    b(1,2)
+
+def call3():
+    b(1,2,3)
+
+def call4():
+    b(1,2,3,4)
+
+# the called function:
+
+cdef b(a, b, c=1, d=2):
+    pass

tests/run/cfuncdef.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test()
 """
 

tests/run/cintop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> int2 = 42
     >>> int3 = 7
     >>> char1 = ord('C')
@@ -10,7 +10,7 @@ __doc__ = """
     >>> int1 ^= int2 >> int3
     >>> int1 ^= int2 << int3 | int2 >> int3
     >>> long1 = char1 | int1
-    >>> print (int1, long1) == f()
+    >>> (int1, long1) == f()
     True
 
     >>> f()

tests/run/classkwonlyargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> spam = Spam()
     >>> b,c,d,e,f,g,h,k = spam.b,spam.c,spam.d,spam.e,spam.f,spam.g,spam.h,spam.k
 
@@ -49,7 +49,7 @@ __doc__ = """
     >>> g(1,2, c=1, e=0, f=2, d=11)
     >>> g(1,2, c=1, f=2, e=0, x=25)
 
-    >>> g(1,2,3)
+    >>> g(1,2,3) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes at most 3 positional arguments (4 given)
     >>> g(1,2)
@@ -84,6 +84,14 @@ __doc__ = """
     TypeError: required keyword argument 'f' is missing
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__, re.M)
+
 class Spam:
     def b(self, a, b, c):
         pass

tests/run/classmethod.pyx

+__doc__ = u"""
+>>> class1.view()
+class1
+>>> class1.plus(1)
+6
+>>> class2.view()
+class2
+>>> class2.plus(1)
+7
+>>> class3.view()
+class3
+>>> class3.plus(1)
+8
+"""
+
+def f_plus(cls, a):
+    return cls.a + a
+
+class class1:
+    a = 5
+    plus = classmethod(f_plus)
+    def view(cls):
+        print cls.__name__
+    view = classmethod(view)
+
+class class2(object):
+    a = 6
+    plus = classmethod(f_plus)
+    def view(cls):
+        print cls.__name__
+    view = classmethod(view)
+
+cdef class class3:
+    a = 7
+    plus = classmethod(f_plus)
+    def view(cls):
+        print cls.__name__
+    view = classmethod(view)

tests/run/classpass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Spam()
-    >>> print s.__class__.__name__
-    Spam
+    >>> s.__class__.__name__
+    'Spam'
 
     >>> s = SpamT()
-    >>> print type(s).__name__
-    SpamT
+    >>> type(s).__name__
+    'SpamT'
 """
 
 class Spam: pass

tests/run/compiledef.pyx

-__doc__ = """
+__doc__ = u"""
     >>> t
     True
     >>> f

tests/run/concatcstrings.pyx

-__doc__ = """
-    >>> spam == "C string 1" + "C string 2"
+__doc__ = u"""
+    >>> spam == u'C string 1' + u'C string 2'
     True
 """
 
-spam = "C string 1" + "C string 2"
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+
+spam = u"C string 1" + u"C string 2"
 

tests/run/cstringmeth.pyx

+__doc__ = u"""
+>>> y
+(b'1', b'2', b'3')
+>>> x
+b'1foo2foo3'
+"""
+
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u"b'", u"'")
+
+
+y = ('1','2','3')
+
+x = 'foo'.join(y)

tests/run/cstringmul.pyx

-__doc__ = """
-    >>> print spam
+__doc__ = u"""
+    >>> print(spam)
     eggseggseggseggs
-    >>> print grail
+    >>> print(grail)
     tomatotomatotomatotomatotomatotomatotomato
 """
 
-spam = "eggs" * 4
-grail = 7 * "tomato"
-
+spam = u"eggs" * 4
+grail = 7 * u"tomato"

tests/run/cstruct.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test_i()
     >>> test_c()
     >>> test_p()

tests/run/ct_DEF.pyx

-__doc__ = """
+__doc__ = u"""
     >>> c()
     120
     >>> i0() == -1
@@ -16,7 +16,7 @@ __doc__ = """
     >>> f()
     12.5
     >>> s()
-    'spam'
+    b'spam'
     >>> two()
     2
     >>> five()
@@ -27,7 +27,15 @@ __doc__ = """
     False
 """
 
-DEF TUPLE = (1, 2, "buckle my shoe")
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" 042", u" 0o42")
+
+DEF TUPLE = (1, 2, u"buckle my shoe")
 DEF TRUE_FALSE = (True, False)
 
 DEF CHAR = c'x'

tests/run/ct_IF.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     1
     >>> g()

tests/run/cunion.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test_i()
     >>> test_c()
     >>> test_p()

tests/run/cvardef.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
 """
 

tests/run/dict.pyx

-__doc__ = """
+__doc__ = u"""
     >>> empty()
     {}
     >>> keyvalue(1, 2)
@@ -11,9 +11,9 @@ __doc__ = """
 
     >>> len(constant())
     2
-    >>> constant()['parrot']
-    'resting'
-    >>> constant()['answer']
+    >>> print(constant()['parrot'])
+    resting
+    >>> print(constant()['answer'])
     42
 """
 
@@ -34,6 +34,5 @@ def keyvalues2(key1, value1, key2, value2):
     return d
 
 def constant():
-    d = {"parrot":"resting", "answer":42}
+    d = {u"parrot":u"resting", u"answer":42}
     return d
-    

tests/run/dietachmayer1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test()
     1.0
 """

tests/run/exarkun.pyx

-__doc__ = """
+__doc__ = u"""
     >>> p = Point(1,2,3)
     >>> p.gettuple()
     (1.0, 2.0, 3.0)

tests/run/extclassbody.pyx

+__doc__ = u"""
+>>> s = Spam()
+>>> s.a
+2
+>>> s.c
+3
+>>> s.test(5)
+13
+>>> s.b
+5
+"""
+
+cdef class Spam:
+    a = 1
+    def test(self, a):
+        return a + self.b + self.c
+    b = a + 2 # 3
+    a = b - 1 # 2
+    c = 3     # 3
+    b = c + a # 5

tests/run/extclasspass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> e = Eggs()
-    >>> print type(e).__name__
-    Eggs
+    >>> type(e).__name__
+    'Eggs'
 """
 
 cdef class Eggs: pass

tests/broken/extcmethod.pyx → tests/run/extcmethod.pyx

+__doc__ = u"""
+>>> test()
+5
+0
+20
+5
+"""
+
 cdef class Spam:
 
     cdef int tons
@@ -8,8 +16,27 @@ cdef class Spam:
     cdef void eat(self):
         self.tons = 0
 
+    def lift(self):
+        print self.tons
 
 cdef class SuperSpam(Spam):
 
     cdef void add_tons(self, int x):
         self.tons = self.tons + 2 * x
+
+def test():
+    cdef Spam s
+    cdef SuperSpam ss
+    s = Spam()
+    s.eat()
+    s.add_tons(5)
+    s.lift()
+
+    ss = SuperSpam()
+    ss.eat()
+    ss.lift()
+
+    ss.add_tons(10)
+    ss.lift()
+
+    s.lift()

tests/run/extinherit.pyx

-__doc__ = """
+__doc__ = u"""
     >>> p = create()
     >>> rest(p)
     0

tests/run/extinstantiate.pyx

-__doc__ = """
-    >>> print type(f()).__name__
-    Spam
+__doc__ = u"""
+    >>> type(f()).__name__
+    'Spam'
 """
 
 cdef class Spam:

tests/run/extkwonlyargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> ext = Ext()
     >>> b,c,d,e,f,g,h,k = ext.b,ext.c,ext.d,ext.e,ext.f,ext.g,ext.h,ext.k
 
@@ -84,6 +84,10 @@ __doc__ = """
     TypeError: required keyword argument 'f' is missing
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
 cdef class Ext:
     def b(self, a, b, c):
         pass

tests/run/extlen.pyx

-__doc__ = """
+__doc__ = u"""
     >>> len(Spam())
     0
 """

tests/broken/extpropertyref.pyx → tests/run/extpropertyref.pyx

+__doc__ = u"""
+>>> tomato()
+42
+"""
+
 cdef class Spam:
 
     property eggs:
     
         def __get__(self):
-            pass
+            return 42
 
-cdef void tomato():
+def tomato():
     cdef Spam spam
     cdef object lettuce
+    spam = Spam()
     lettuce = spam.eggs
-
+    return lettuce

tests/run/extstarargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Silly(1,2,3, 'test')
     >>> (spam,grail,swallow,creosote,onlyt,onlyk,tk) = (
     ...     s.spam,s.grail,s.swallow,s.creosote,s.onlyt,s.onlyk,s.tk)
 
     >>> spam(1,2,3)
     (1, 2, 3)
-    >>> spam(1,2)
+    >>> spam(1,2) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (2 given)
     >>> spam(1,2,3,4)
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (4 given)
-    >>> spam(1,2,3, a=1)
+    >>> spam(1,2,3, a=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: 'a' is an invalid keyword argument for this function
 
@@ -21,10 +21,10 @@ __doc__ = """
     (1, 2, 3, (4,))
     >>> grail(1,2,3,4,5,6,7,8,9)
     (1, 2, 3, (4, 5, 6, 7, 8, 9))
-    >>> grail(1,2)
+    >>> grail(1,2) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (2 given)
-    >>> grail(1,2,3, a=1)
+    >>> grail(1,2,3, a=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: 'a' is an invalid keyword argument for this function
 
@@ -35,7 +35,7 @@ __doc__ = """
     TypeError: function takes at most 3 positional arguments (4 given)
     >>> swallow(1,2,3, a=1, b=2)
     (1, 2, 3, (('a', 1), ('b', 2)))
-    >>> swallow(1,2,3, x=1)
+    >>> swallow(1,2,3, x=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: keyword parameter 'x' was given by position and by name
 
@@ -47,7 +47,7 @@ __doc__ = """
     (1, 2, 3, (), (('a', 1),))
     >>> creosote(1,2,3,4, a=1, b=2)
     (1, 2, 3, (4,), (('a', 1), ('b', 2)))
-    >>> creosote(1,2,3,4, x=1)
+    >>> creosote(1,2,3,4, x=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: keyword parameter 'x' was given by position and by name
 
@@ -88,8 +88,16 @@ __doc__ = """
     (1, ('a', 1), ('b', 2))
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__, re.M)
+
 cdef sorteditems(d):
-    l = d.items()
+    l = list(d.items())
     l.sort()
     return tuple(l)
 

tests/run/exttype.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Spam(12)
     >>> s.eat()
     12 42

tests/run/filenames.pxi

+spam = u"ftang"

tests/broken/filenames.pyx → tests/run/filenames.pyx

+__doc__ = u"""
+>>> print(spam)
+ftang
+>>> foo
+42
+"""
+
 include "filenames.pxi"
 
 foo = 42

tests/run/future_unicode_literals.pyx

+from __future__ import unicode_literals
+
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = """
+    >>> u == 'test'
+    True
+    >>> isinstance(u, str)
+    True
+"""
+else:
+    __doc__ = """
+    >>> u == u'test'
+    True
+    >>> isinstance(u, unicode)
+    True
+"""
+
+u = "test"

tests/run/getattr3call.pyx

-__doc__ = """
+__doc__ = u"""
     >>> class test(object): a = 1
     >>> t = test()
 

tests/run/if.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(0,0)
     0
     >>> f(1,2)

tests/run/include.pyx

-__doc__ = """
+__doc__ = u"""
     >>> D
     2
 """

tests/run/inhcmethcall.pyx

+__doc__ = u"""
+>>> p = Norwegian()
+>>> p.describe()
+Norwegian
+Parrot
+"""
+
+cdef class Parrot:
+
+  cdef void _describe(self):
+    print u"Parrot"
+
+  def describe(self):
+    self._describe()
+
+cdef class Norwegian(Parrot):
+
+  cdef void _describe(self):
+    print u"Norwegian"
+    Parrot._describe(self)

tests/run/inop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1,[1,2,3])
     True
     >>> f(5,[1,2,3])

tests/run/ishimoto2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> C().xxx(5)
     5
     >>> C().xxx()
-    'a b'
+    u'a b'
     >>> C().xxx(42)
     42
     >>> C().xxx()
-    'a b'
+    u'a b'
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+
 class C:
-    def xxx(self, p="a b"):
+    def xxx(self, p=u"a b"):
         return p

tests/run/ishimoto3.pyx

-__doc__ = """
+__doc__ = u"""
     >>> c1 = C1()
     >>> c2 = C2(c1)
     >>> c1 is c2.getc1()

tests/run/isnonebool.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test_and(None, None)
     True
     >>> test_and(None, 1)

tests/run/jarausch1.pyx

-__doc__ = """
-   >>> py_x = r'\\\\'
+__doc__ = u"""
+   >>> py_x = br'\\\\'
    >>> assert x == py_x
 """
 
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" br'", u" r'")
+
 x = r'\\'

tests/broken/king1.pyx → tests/run/king1.pyx

+__doc__ = u"""
+>>> f()
+It works!
+"""
+
 DEF STUFF = "Spam"
 
-cdef void f():
+def f():
     IF STUFF == "Spam":
-        print "It works!"
+        print u"It works!"
     ELSE:
-        print "Doesn't work"
+        print u"Doesn't work"

tests/run/kostyrka.pyx

-__doc__ = """
+__doc__ = u"""
     >>> t = TEST()
     >>> 1 in t
     True

tests/run/kostyrka2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> x = X()
     >>> x.slots
-    ['']
+    [b'']
 """
 
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u"b'", u"'")
+
 class X:
         slots = ["", ]

tests/run/kwargproblems.pyx

-__doc__ = """
+__doc__ = u"""
     >>> d = {1 : 2}
     >>> test(**d)
     Traceback (most recent call last):
@@ -12,13 +12,18 @@ __doc__ = """
     >>> d
     {}
 
-    >>> d = {'arg' : 2}
+    >>> d = {'arg' : 2} # this should be u'arg', but Py2 can't handle it...
     >>> test(**d)
     {'arg': 3}
     >>> d
     {'arg': 2}
 """
 
+import sys
+
 def test(**kw):
-    kw['arg'] = 3
+    if sys.version_info[0] >= 3:
+        kw[u'arg'] = 3
+    else:
+        kw['arg'] = 3
     return kw

tests/run/kwonlyargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> b(1,2,3)
     >>> b(1,2,3,4)
     Traceback (most recent call last):
@@ -81,6 +81,10 @@ __doc__ = """
     TypeError: required keyword argument 'f' is missing
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
 def b(a, b, c):
     pass
 

tests/run/kwonlyargscall.pyx

+__doc__ = u"""
+    >>> call3(b)
+    >>> call4(b)
+    Traceback (most recent call last):
+    TypeError: function takes exactly 3 arguments (4 given)
+
+    >>> call2(c)
+    >>> call3(c)
+    >>> call4(c)
+    Traceback (most recent call last):
+    TypeError: function takes at most 3 arguments (4 given)
+
+    >>> call2(d)
+    >>> call2c(d)
+
+    >>> call3(d)
+    Traceback (most recent call last):
+    TypeError: function takes at most 2 positional arguments (3 given)
+    >>> call2d(d)
+    Traceback (most recent call last):
+    TypeError: 'd' is an invalid keyword argument for this function
+
+    >>> call2(e)
+    >>> call2c(e)
+    >>> call2d(e)
+    >>> call2cde(e)
+    >>> call3(e)
+    >>> call4(e)
+    Traceback (most recent call last):
+    TypeError: function takes at most 3 positional arguments (4 given)
+
+    >>> call2c(f)
+    >>> call2cd(f)
+
+    >>> call3(f)
+    Traceback (most recent call last):
+    TypeError: function takes at most 2 positional arguments (3 given)
+    >>> call2(f)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'c' is missing
+    >>> call2ce(f)
+    Traceback (most recent call last):
+    TypeError: 'e' is an invalid keyword argument for this function
+
+    >>> call2cf(g)
+    >>> call2cefd(g)
+    >>> call2cfex(g)
+
+    >>> call3(g)
+    Traceback (most recent call last):
+    TypeError: function takes at most 2 positional arguments (3 given)
+    >>> call2(g)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'c' is missing
+    >>> call2c(g)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'f' is missing
+
+    >>> call2cf(h)
+    >>> call2cfe(h)
+    >>> call6cf(h)
+    >>> call6cfexy(h)
+
+    >>> call3(h)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'c' is missing
+    >>> call3d(h)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'c' is missing
+
+    >>> call2cf(k)
+    >>> call2cfe(k)
+    >>> call6df(k)
+    >>> call6dfexy(k)
+
+    >>> call3(k)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'f' is missing
+    >>> call2d(k)
+    Traceback (most recent call last):
+    TypeError: required keyword argument 'f' is missing
+"""
+
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
+# the calls:
+
+def call2(f):
+    f(1,2)
+
+def call3(f):
+    f(1,2,3)
+
+def call4(f):
+    f(1,2,3,4)
+
+def call2c(f):
+    f(1,2, c=1)
+
+def call2d(f):
+    f(1,2, d=1)
+
+def call3d(f):
+    f(1,2,3, d=1)
+
+def call2cd(f):
+    f(1,2, c=1, d=2)
+
+def call2ce(f):
+    f(1,2, c=1, e=2)
+
+def call2cde(f):
+    f(1,2, c=1, d=2, e=3)
+
+def call2cf(f):
+    f(1,2, c=1, f=2)
+
+def call6cf(f):
+    f(1,2,3,4,5,6, c=1, f=2)
+
+def call6df(f):
+    f(1,2,3,4,5,6, d=1, f=2)
+
+def call2cfe(f):
+    f(1,2, c=1, f=2, e=3)
+
+def call2cefd(f):
+    f(1,2, c=1, e=0, f=2, d=11)
+
+def call2cfex(f):
+    f(1,2, c=1, f=2, e=0, x=25)
+
+def call6cfexy(f):
+    f(1,2,3,4,5,6, c=1, f=2, e=3, x=25, y=11)
+
+def call6dfexy(f):
+    f(1,2,3,4,5,6, d=1, f=2, e=3, x=25, y=11)
+
+# the called functions:
+
+def b(a, b, c):
+    pass
+
+def c(a, b, c=1):
+    pass
+
+def d(a, b, *, c = 88):
+    pass
+
+def e(a, b, c = 88, **kwds):
+    pass
+
+def f(a, b, *, c, d = 42):
+    pass
+
+def g(a, b, *, c, d = 42, e = 17, f, **kwds):
+    pass
+
+def h(a, b, *args, c, d = 42, e = 17, f, **kwds):
+    pass
+
+def k(a, b, c=1, *args, d = 42, e = 17, f, **kwds):
+    pass

tests/run/lepage_1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> a = A(1,2,3)
     >>> a[0]
     1.0

tests/run/list.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1, 2, 3, 4, 5)
     []
     >>> g(1, 2, 3, 4, 5)

tests/run/literals.pyx

-__doc__ = """
+__doc__ = u"""
     >>> foo()
 """
 

tests/broken/menten1.pyx → tests/run/menten1.pyx

-cdef loops():
+__doc__ = u"""
+>>> loops()
+5
+"""
+
+def loops():
     cdef int k
     for i from 0 <= i < 5:
         for j from 0 <= j < 2:
             k = i + j
+    return k

tests/run/modbody.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     >>> g
     42
     >>> x
-    'spam'
+    u'spam'
     >>> y
-    'eggs'
+    u'eggs'
     >>> z
-    'spameggs'
+    u'spameggs'
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+
 def f():
     pass
     
 g = 42
-x = "spam"
-y = "eggs"
+x = u"spam"
+y = u"eggs"
 if g:
     z = x + y
-

tests/run/modop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> modobj(9,2)
     1
     >>> modobj('%d', 5)

tests/run/multass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     (1, 2, 1, 2)
     >>> g()
     (1, 1, 2, 2, 3, 3)
     >>> h()
-    (1, 'test', 3, 1, 'test', 3)
+    (1, b'test', 3, 1, b'test', 3)
     >>> j()
     (2, 1, 4, 2, 6, 3)
 """
 
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 def f():
     cdef object obj1a, obj2a, obj3a, obj1b, obj2b, obj3b
     obj1b, obj2b, obj3b = 1, 2, 3

tests/run/new_style_exceptions.pyx

-__doc__ = """
-    >>> test(Exception('hi'))
-    Raising: Exception('hi',)
-    Caught: Exception('hi',)
+__doc__ = u"""
+    >>> test(Exception(u'hi'))
+    Raising: Exception(u'hi',)
+    Caught: Exception(u'hi',)
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"u'", u"'")
+
 import sys, types
 
 def test(obj):
-    print "Raising: %s%r" % (obj.__class__.__name__, obj.args)
+    print u"Raising: %s%r" % (obj.__class__.__name__, obj.args)
     try:
         raise obj
     except:
@@ -16,4 +20,4 @@ def test(obj):
             assert isinstance(info[0], type)
         else:
             assert isinstance(info[0], types.ClassType)
-        print "Caught: %s%r" % (info[1].__class__.__name__, info[1].args)
+        print u"Caught: %s%r" % (info[1].__class__.__name__, info[1].args)

tests/broken/nononetypecheck.pyx → tests/run/nononetypecheck.pyx

+__doc__ = u"""
+>>> g()
+"""
+
 cdef class Spam:
     pass
 
 cdef f(Spam s):
     pass
 
-cdef g():
+def g():
     f(None)

tests/run/notinop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1,[1,2,3])
     False
     >>> f(5,[1,2,3])

tests/run/pass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
 """
 

tests/run/pinard5.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test()
     1
 """

tests/run/pinard6.pyx

-__doc__ = """
+__doc__ = u"""
     >>> x
     (1, 2)
 """

tests/run/pinard7.pyx

-__doc__ = """
+__doc__ = u"""
     >>> c = build()
     >>> c.method()
     Traceback (most recent call last):

tests/run/pinard8.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f = Fiche()
     >>> f[0] = 1
     >>> f.geti()

tests/run/powop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1.0, 2.95)[0] == f(1.0, 2.95)[1]
     True
 
@@ -15,6 +15,10 @@ __doc__ = """
     True
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"2L", u"2")
+
 def f(obj2, obj3):
     cdef float flt1, flt2, flt3
     flt2, flt3 = obj2, obj3

tests/run/print.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1, 'test')
     <BLANKLINE>
     1
@@ -11,5 +11,4 @@ def f(a, b):
     print a
     print a, b
     print a, b,
-    print 42, "spam"
-    
+    print 42, u"spam"

tests/run/pycmp.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     >>> g()
 """

tests/run/pyextattrref.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Spam(Eggs("ham"))
     >>> test(s)
     'ham'

tests/run/pyintop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1,2,3)
     3
     >>> g(1,2,3)

tests/run/pylistsubtype.pyx

-__doc__ = """
+__doc__ = u"""
     >>> l1 = Sub1([1,2,3])
     >>> len(l1)
     3

tests/run/pynumop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     6
     >>> g()
-    0
+    2
 """
 
 def f():
@@ -13,8 +13,8 @@ def f():
     return obj1
 
 def g():
-    obj1 = 1
-    obj2 = 2
+    obj1 = 12
+    obj2 = 6
     obj3 = 3
     obj1 = obj2 / obj3
-    return obj1
+    return int(obj1)

tests/run/r_addint.pyx

-__doc__ = """
+__doc__ = u"""
     >>> def test(a, b):
-    ...     print a, b, add(a, b)
+    ...     return (a, b, add(a, b))
 
     >>> test(1, 2)
-    1 2 3
-    >>> test(17.3, 88.6)
-    17.3 88.6 105.9
-    >>> test("eggs", "spam")
-    eggs spam eggsspam
+    (1, 2, 3)
+    >>> [ str(f) for f in test(17.3, 88.6) ]
+    ['17.3', '88.6', '105.9']
+    >>> test(u'eggs', u'spam')
+    (u'eggs', u'spam', u'eggsspam')
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"u'", u"'")
+
 def add(x, y):
     return x + y

tests/run/r_argdefault.pyx

-__doc__ = """
+__doc__ = u"""
   >>> swallow(name = "Brian")
   This swallow is called Brian
   >>> swallow(airspeed = 42)
@@ -9,9 +9,9 @@ __doc__ = """
 
 def swallow(name = None, airspeed = None, coconuts = None):
     if name is not None:
-        print "This swallow is called", name
+        print u"This swallow is called", name
     if airspeed is not None:
-        print "This swallow is flying at", airspeed, "furlongs per fortnight"
+        print u"This swallow is flying at", airspeed, u"furlongs per fortnight"
     if coconuts is not None:
-        print "This swallow is carrying", coconuts, "coconuts"
+        print u"This swallow is carrying", coconuts, u"coconuts"
 

tests/run/r_barbieri1.pyx

-__doc__ = """
+__doc__ = u"""
   >>> try:
   ...     B()
   ... except Exception, e:
-  ...     print "%s: %s" % (e.__class__.__name__, e)
+  ...     print("%s: %s" % (e.__class__.__name__, e))
   Exception: crash-me
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"Exception, e", u"Exception as e")
+
 cdef class A:
     def __cinit__(self):
-        raise Exception("crash-me")
+        raise Exception(u"crash-me")
 
 cdef class B(A):
     def __cinit__(self):

tests/run/r_bishop3.pyx

-__doc__ = """
-foo = Foo()
-fee = Fee()
-faa = Faa()
-fee.bof()
-faa.bof()
+__doc__ = u"""
+>>> foo = Foo()
+>>> fee = Fee()
+>>> faa = Faa()
+>>> fee.bof()
+Fee bof 0
+>>> faa.bof()
+Foo bof 0
 """
 
 cdef class Foo:
@@ -16,10 +18,10 @@ cdef class Foo:
 cdef class Fee(Foo):
 
   def bof(self):
-    print 'Fee bof', self.val
+    print u'Fee bof', self.val
 
 
 cdef class Faa(Fee):
 
   def bof(self):
-    print 'Foo bof', self.val
+    print u'Foo bof', self.val

tests/run/r_bowden1.pyx

-__doc__ = """
-print f(100)
-print g(3000000000)
+__doc__ = u"""
+>>> f(100)
+101L
+>>> g(3000000000)
+3000000001L
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"L", u"")
+
 def f(x):
     cdef unsigned long long ull
     ull = x

tests/run/r_delgado_1.pyx

-__doc__ = """
-try:
-    eggs().eat()
-except RuntimeError, e:
-    print "%s: %s" % (e.__class__.__name__, e)
+__doc__ = u"""
+>>> try:
+...     eggs().eat()
+... except RuntimeError:
+...    import sys
+...    e = sys.exc_info()[1]
+...    print("%s: %s" % (e.__class__.__name__, e))
+RuntimeError: I don't like that
 """
 
 cdef class eggs:
@@ -11,5 +14,5 @@ cdef class eggs:
     pass
 
   def eat(self):
-    raise RuntimeError("I don't like that")
+    raise RuntimeError(u"I don't like that")
 

tests/run/r_docstrings.pyx

-__doc__ = """
+__doc__ = u"""# Python 3 gets all of these right ...
     >>> f.__doc__
     'This is a function docstring.'
 
     >>> C.__doc__
-    'This is a class docstring.'
+    u'This is a class docstring.'
     >>> CS.__doc__
-    'This is a subclass docstring.'
-    >>> print CSS.__doc__
+    u'This is a subclass docstring.'
+    >>> print(CSS.__doc__)
     None
 
     >>> T.__doc__
@@ -18,41 +18,45 @@ __doc__ = """
 Compare with standard Python:
 
     >>> def f():
-    ...     'This is a function docstring.'
+    ...     u'This is a function docstring.'
     >>> f.__doc__
-    'This is a function docstring.'
+    u'This is a function docstring.'
 
     >>> class C:
-    ...     'This is a class docstring.'
+    ...     u'This is a class docstring.'
     >>> class CS(C):
-    ...     'This is a subclass docstring.'
+    ...     u'This is a subclass docstring.'
     >>> class CSS(CS):
     ...     pass
 
     >>> C.__doc__
-    'This is a class docstring.'
+    u'This is a class docstring.'
     >>> CS.__doc__
-    'This is a subclass docstring.'
+    u'This is a subclass docstring.'
     >>> CSS.__doc__
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+
 def f():
-    "This is a function docstring."
+    u"This is a function docstring."
 
 class C:
-    "This is a class docstring."
+    u"This is a class docstring."
 
 class CS(C):
-    "This is a subclass docstring."
+    u"This is a subclass docstring."
 
 class CSS(CS):
     pass
 
 cdef class T:
-    "This is an extension type docstring."
+    u"This is an extension type docstring."
 
 cdef class TS(T):
-    "This is an extension subtype docstring."
+    u"This is an extension subtype docstring."
 
 cdef class TSS(TS):
     pass

tests/run/r_extcomplex2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> c = eggs()
-    >>> print "eggs returned:", c
-    eggs returned: (17+42j)
+    >>> c
+    (17+42j)
     >>> spam(c)
     Real: 17.0
     Imag: 42.0
@@ -17,8 +17,8 @@ cdef extern from "complexobject.h":
         cdef Py_complex cval
     
 def spam(complex c):
-    print "Real:", c.cval.real
-    print "Imag:", c.cval.imag
+    print u"Real:", c.cval.real
+    print u"Imag:", c.cval.imag
 
 def eggs():
     return complex(17, 42)

tests/run/r_extstarargs.pyx

-__doc__ = """
+__doc__ = u"""
   >>> s = Swallow("Brian", 42)
   Name: Brian
   Airspeed: 42
@@ -33,7 +33,7 @@ __doc__ = """
 cdef class Swallow:
 
     def __init__(self, name, airspeed, *args, **kwds):
-        print "Name:", name
-        print "Airspeed:", airspeed
-        print "Extra args:", args
-        print "Extra keywords:", kwds
+        print u"Name:", name
+        print u"Airspeed:", airspeed
+        print u"Extra args:", args
+        print u"Extra keywords:", kwds

tests/run/r_forloop.pyx

-__doc__ = """
+__doc__ = u"""
   >>> go()
   Spam!
   Spam!
@@ -9,5 +9,5 @@ __doc__ = """
 
 def go():
     for i in range(5):
-        print "Spam!"
+        print u"Spam!"
 

tests/run/r_hordijk1.pyx

-__doc__ = """
+__doc__ = u"""
   >>> try:
   ...     s = Spam()
-  ... except StandardError, e:
-  ...     print "Exception:", e
+  ... except KeyError, e:
+  ...     print("Exception: %s" % e)
   ... else:
-  ...     print "Did not raise the expected exception"
-  Exception: This is not a spanish inquisition
+  ...     print("Did not raise the expected exception")
+  Exception: u'This is not a spanish inquisition'
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"Error, e", u"Error as e")
+    __doc__ = __doc__.replace(u" u'", u" '")
+
 cdef extern from "Python.h":
-    ctypedef class types.ListType [object PyListObject]:
+    ctypedef class __builtin__.list [object PyListObject]:
         pass
 
-cdef class Spam(ListType):
+cdef class Spam(list):
     def __init__(self):
-        raise StandardError("This is not a spanish inquisition")
+        raise KeyError(u"This is not a spanish inquisition")

tests/run/r_huss3.pyx

-__doc__ = """
-try:
-    foo()
-except Exception, e:
-    print "%s: %s" % (e.__class__.__name__, e)
+__doc__ = u"""
+>>> try:
+...     foo()
+... except Exception, e:
+...     print("%s: %s" % (e.__class__.__name__, e))
+ValueError: 
+>>> try:
+...     bar()
+... except Exception, e:
+...     print("%s: %s" % (e.__class__.__name__, e))
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"Exception, e", u"Exception as e")
+
 def bar():
     try:
         raise TypeError

tests/run/r_jeff_epler_1.pyx

-__doc__ = """
-print r_jeff_epler_1.blowup([2, 3, 5])
+__doc__ = u"""
+    >>> blowup([2, 3, 5])
+    1
 """
 
 def blowup(p):

tests/run/r_jiba1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test()
     This parrot is resting.
     Lovely plumage!
@@ -8,7 +8,7 @@ __doc__ = """
 cdef class Parrot:
 
     cdef void describe(self):
-        print "This parrot is resting."
+        print u"This parrot is resting."
 
     def describe_python(self):
         self.describe()
@@ -16,7 +16,7 @@ cdef class Parrot:
 cdef class Norwegian(Parrot):
 
     cdef void describe(self):
-        print "Lovely plumage!"
+        print u"Lovely plumage!"
 
 def test():
     cdef Parrot p1, p2

tests/run/r_lepage_3.pyx

-__doc__ = """
-g = r_lepage_3.Grail()
-g("spam", 42, ["tomato", "sandwich"])
+__doc__ = u"""
+>>> g = Grail()
+>>> g("spam", 42, ["tomato", "sandwich"])
+Grail called with: spam 42 ['tomato', 'sandwich']
 """
 
 cdef class Grail:
 
     def __call__(self, x, y, z):
-        print "Grail called with:", x, y, z
+        print u"Grail called with:", x, y, z

tests/run/r_mang1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> import re
     >>> t
-    ('2',)
+    (u'2',)
     >>> t == re.search('(\\d+)', '-2.80 98\\n').groups()
     True
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"(u'", u"('")
+
 # this is a string constant test, not a test for 're'
 
 import re
-t = re.search('(\d+)', '-2.80 98\n').groups()
+t = re.search(u'(\d+)', u'-2.80 98\n').groups()

tests/run/r_mcintyre1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> b = Bicycle()
     >>> b.fall_off()
     Falling off extremely hard
@@ -8,5 +8,5 @@ __doc__ = """
 
 class Bicycle:
 
-    def fall_off(self, how_hard = "extremely"):
-        print "Falling off", how_hard, "hard"
+    def fall_off(self, how_hard = u"extremely"):
+        print u"Falling off", how_hard, u"hard"

tests/run/r_mitch_chapman_2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> boolExpressionsFail()
-    'Not 2b'
+    u'Not 2b'
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+
 def boolExpressionsFail():
     dict = {1: 1}
-    if not dict.has_key("2b"):
-        return "Not 2b"
+    if not "2b" in dict:
+        return u"Not 2b"
     else:
-        return "2b?"
+        return u"2b?"

tests/run/r_primes.pyx

-__doc__ = """
-    >>> print primes(20)
+__doc__ = u"""
+    >>> primes(20)
     [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71]
 """
 

tests/run/r_print.pyx

-__doc__ = """
+__doc__ = u"""
   >>> frighten()
   NOBODY expects the Spanish Inquisition!
 """
 
 def frighten():
-    print "NOBODY", "expects", "the Spanish Inquisition!"
+    print u"NOBODY", u"expects", u"the Spanish Inquisition!"
 

tests/run/r_pyclass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> order()
     42 tons of spam!
 """
@@ -9,7 +9,7 @@ class Spam:
         self.weight = w
 
     def serve(self):
-        print self.weight, "tons of spam!"
+        print self.weight, u"tons of spam!"
 
 def order():
     s = Spam(42)

tests/run/r_pyclassdefault.pyx

-__doc__ = """
+__doc__ = u"""
   >>> c = CoconutCarrier()
   >>> c.swallow(name = "Brian")
   This swallow is called Brian
@@ -12,8 +12,8 @@ class CoconutCarrier:
 
     def swallow(self, name = None, airspeed = None, coconuts = None):
         if name is not None:
-            print "This swallow is called", name
+            print u"This swallow is called", name
         if airspeed is not None:
-            print "This swallow is flying at", airspeed, "furlongs per fortnight"
+            print u"This swallow is flying at", airspeed, u"furlongs per fortnight"
         if coconuts is not None:
-            print "This swallow is carrying", coconuts, "coconuts"
+            print u"This swallow is carrying", coconuts, u"coconuts"

tests/run/r_pythonapi.pyx

-__doc__ = """
+__doc__ = u"""
     >>> x = spam()
-    >>> print repr(x)
-    'Ftang\\x00Ftang!'
+    >>> print(repr(x))
+    b'Ftang\\x00Ftang!'
 """
 
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 cdef extern from "string.h":
     void memcpy(char *d, char *s, int n)
 

tests/run/r_spamtype.pyx

-__doc__ = """
+__doc__ = u"""
     >>> s = Spam()
-    >>> print s.get_tons()
+    >>> s.get_tons()
     17
     >>> s.set_tons(42)
-    >>> print s.get_tons()
+    >>> s.get_tons()
     42
     >>> s = None
     42 tons of spam is history.
@@ -17,7 +17,7 @@ cdef class Spam:
         self.tons = 17
     
     def __dealloc__(self):
-        print self.tons, "tons of spam is history."
+        print self.tons, u"tons of spam is history."
     
     def get_tons(self):
         return self.tons

tests/run/r_starargcall.pyx

-__doc__ = """
+__doc__ = u"""
     >>> eggs()
     Args: 1 2 3
     Args: buckle my shoe
 """
 
 def spam(a, b, c):
-    print "Args:", a, b, c
+    print u"Args:", a, b, c
 
 def eggs():
     spam(*(1,2,3))
-    spam(*["buckle","my","shoe"])
+    spam(*[u"buckle",u"my",u"shoe"])
 

tests/run/r_starargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> swallow("Brian", 42)
     Name: Brian
     Airspeed: 42
@@ -31,8 +31,8 @@ __doc__ = """
 """
 
 def swallow(name, airspeed, *args, **kwds):
-    print "Name:", name
-    print "Airspeed:", airspeed
-    print "Extra args:", args
-    print "Extra keywords:", kwds
+    print u"Name:", name
+    print u"Airspeed:", airspeed
+    print u"Extra args:", args
+    print u"Extra keywords:", kwds
 

tests/run/r_starargsonly.pyx

-__doc__ = """
+__doc__ = u"""
     >>> spam()
     Args: ()
     >>> spam(42)
@@ -8,5 +8,5 @@ __doc__ = """
 """
 
 def spam(*args):
-    print "Args:", args
+    print u"Args:", args
 

tests/run/r_toofewargs.pyx

-__doc__ = """
-    >>> s = Spam()
+__doc__ = u"""
+    >>> s = Spam() #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (0 given)
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: .*", u"Error: ...", __doc__)
+
 cdef class Spam:
 
     def __init__(self, a, b, int c):

tests/run/r_vree_1.pyx

-__doc__ = """
+__doc__ = """# disabled in Py3
     >>> test(0)
     0L
     >>> test(1)

tests/run/ref2local.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     42
 """

tests/run/return.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f('test')
     >>> test_g()
     >>> test_h(5)

tests/run/rodriguez_1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> b = B()
     >>> b.t
     {1: ((1, 2, 3),), 2: (1, 2, 3)}

tests/run/simpcall.pyx

-__doc__ = """
-    >>> z(1,9.2,'test')
+__doc__ = u"""
+    >>> z(1,9.2, b'test')
     >>> failtype()
     Traceback (most recent call last):
     TypeError: an integer is required
 
-    >>> fail0(1,2)
+    >>> fail0(1,2) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 2 arguments (0 given)
 
-    >>> fail1(1,2)
+    >>> fail1(1,2) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 2 arguments (1 given)
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: .*exactly.*", u"Error: ...", __doc__)
+
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 def f(x, y):
     x = y
 

tests/run/sizeof.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
 """
 

tests/run/slice2.pyx

-__doc__ = """
+__doc__ = u"""
     >>> l = [1,2,3,4]
 
     >>> f(1, l, 2, 3)

tests/run/slice3.pyx

-__doc__ = """
+__doc__ = u"""
     >>> class Test(object):
     ...     def __setitem__(self, key, value):
-    ...         print key, value
+    ...         print((key, value))
     ...     def __getitem__(self, key):
-    ...         print key
+    ...         print(key)
     ...         return self
 
     >>> ellipsis(Test())
@@ -23,7 +23,7 @@ __doc__ = """
     slice(1, 2, 3)
 
     >>> set(Test(), -11)
-    slice(1, 2, 3) -11
+    (slice(1, 2, 3), -11)
 """
 
 def ellipsis(o):

tests/run/specialfloat.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     12.5
 

tests/run/starargs.pyx

-__doc__ = """
+__doc__ = u"""
     >>> spam(1,2,3)
     (1, 2, 3)
-    >>> spam(1,2)
+    >>> spam(1,2) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (2 given)
     >>> spam(1,2,3,4)
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (4 given)
-    >>> spam(1,2,3, a=1)
+    >>> spam(1,2,3, a=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: 'a' is an invalid keyword argument for this function
 
@@ -17,10 +17,10 @@ __doc__ = """
     (1, 2, 3, (4,))
     >>> grail(1,2,3,4,5,6,7,8,9)
     (1, 2, 3, (4, 5, 6, 7, 8, 9))
-    >>> grail(1,2)
+    >>> grail(1,2) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: function takes exactly 3 arguments (2 given)
-    >>> grail(1,2,3, a=1)
+    >>> grail(1,2,3, a=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: 'a' is an invalid keyword argument for this function
 
@@ -31,7 +31,7 @@ __doc__ = """
     TypeError: function takes at most 3 positional arguments (4 given)
     >>> swallow(1,2,3, a=1, b=2)
     (1, 2, 3, (('a', 1), ('b', 2)))
-    >>> swallow(1,2,3, x=1)
+    >>> swallow(1,2,3, x=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: keyword parameter 'x' was given by position and by name
 
@@ -43,7 +43,7 @@ __doc__ = """
     (1, 2, 3, (), (('a', 1),))
     >>> creosote(1,2,3,4, a=1, b=2)
     (1, 2, 3, (4,), (('a', 1), ('b', 2)))
-    >>> creosote(1,2,3,4, x=1)
+    >>> creosote(1,2,3,4, x=1) #doctest: +ELLIPSIS
     Traceback (most recent call last):
     TypeError: keyword parameter 'x' was given by position and by name
 
@@ -84,8 +84,16 @@ __doc__ = """
     (1, ('a', 1), ('b', 2))
 """
 
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"Error: (.*)exactly(.*)", u"Error: \\1at most\\2", __doc__)
+
+import sys, re
+if sys.version_info >= (2,6):
+    __doc__ = re.sub(u"(ELLIPSIS[^>]*Error: )[^\n]*\n", u"\\1...\n", __doc__, re.M)
+
 cdef sorteditems(d):
-    l = d.items()
+    l = list(d.items())
     l.sort()
     return tuple(l)
 

tests/run/staticmethod.pyx

+__doc__ = u"""
+>>> class1.plus1(1)
+2
+>>> class2.plus1(1)
+2
+>>> class3.plus1(1)
+2
+"""
+
+def f_plus(a):
+    return a + 1
+
+class class1:
+    plus1 = f_plus
+
+class class2(object):
+    plus1 = f_plus
+
+cdef class class3:
+    plus1 = f_plus

tests/run/strconstinclass.pyx

-__doc__ = """
+__doc__ = u"""
     >>> c = C()
-    >>> print c.x
-    foo
+    >>> c.x
+    b'foo'
 """
 
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 class C:
     x = "foo"
 

tests/run/strfunction.pyx

-__doc__ = """
-   >>> s('test')
-   'test'
+__doc__ = u"""
+   >>> s('test', **encoding)
+   b'test'
    >>> z
-   'test'
+   b'test'
    >>> c('testing')
-   'testing'
+   b'testing'
    >>> sub('testing a subtype')
-   'testing a subtype'
-   >>> subs('testing a subtype')
-   'testing a subtype'
+   b'testing a subtype'
+   >>> subs('testing a subtype', **encoding)
+   b'testing a subtype'
 
 #   >>> csub('testing a subtype')
 #   'testing a subtype'
@@ -16,20 +16,27 @@ __doc__ = """
 #   'testing a subtype'
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    encoding = {u'encoding' : u'ASCII'}
+else:
+    encoding = {}
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 s = str
 z = str('test')
 
 def c(string):
-    return str(string)
+    return str(string, **encoding)
 
 class subs(str):
     pass
 
 def sub(string):
-    return subs(string)
+    return subs(string, **encoding)
 
 #cdef class subs(str):
 #    pass
 
 #def csub(string):
-#    return csubs(string)
+#    return csubs(string, **encoding)

tests/run/strliterals.pyx

+__doc__ = ur"""
+    >>> s1
+    b'abc\x11'
+    >>> s1 == b'abc\x11'
+    True
+    >>> len(s1)
+    4
+
+    >>> s2
+    b'abc\\x11'
+    >>> s2 == br'abc\x11'
+    True
+    >>> len(s2)
+    7
+
+    >>> s3
+    b'abc\\x11'
+    >>> s3 == bR'abc\x11'
+    True
+    >>> len(s3)
+    7
+
+    >>> s4
+    b'abc\x11'
+    >>> s4 == b'abc\x11'
+    True
+    >>> len(s4)
+    4
+
+    >>> s5
+    b'abc\x11'
+    >>> s5 == B'abc\x11'
+    True
+    >>> len(s5)
+    4
+
+    >>> s6
+    b'abc\\x11'
+    >>> s6 == br'abc\x11'
+    True
+    >>> len(s6)
+    7
+
+    >>> s7
+    b'abc\\x11'
+    >>> s7 == Br'abc\x11'
+    True
+    >>> len(s7)
+    7
+
+    >>> s8
+    b'abc\\x11'
+    >>> s8 == bR'abc\x11'
+    True
+    >>> len(s8)
+    7
+
+    >>> s9
+    b'abc\\x11'
+    >>> s9 == BR'abc\x11'
+    True
+    >>> len(s9)
+    7
+
+    >>> u1
+    u'abc\x11'
+    >>> u1 == u'abc\x11'
+    True
+    >>> len(u1)
+    4
+
+    >>> u2
+    u'abc\x11'
+    >>> u2 == U'abc\x11'
+    True
+    >>> len(u2)
+    4
+
+    >>> u3
+    u'abc\\x11'
+    >>> u3 == ur'abc\x11'
+    True
+    >>> len(u3)
+    7
+
+    >>> u4
+    u'abc\\x11'
+    >>> u4 == Ur'abc\x11'
+    True
+    >>> len(u4)
+    7
+
+    >>> u5
+    u'abc\\x11'
+    >>> u5 == uR'abc\x11'
+    True
+    >>> len(u5)
+    7
+
+    >>> u6
+    u'abc\\x11'
+    >>> u6 == UR'abc\x11'
+    True
+    >>> len(u6)
+    7
+"""
+
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '").replace(u" U'", u" '").replace(u" ur'", u" r'").replace(u" uR'", u" R'").replace(u" Ur'", u" r'").replace(u" UR'", u" R'")
+else:
+    __doc__ = __doc__.replace(u" b'", u" '").replace(u" B'", u" '").replace(u" br'", u" r'").replace(u" bR'", u" R'").replace(u" Br'", u" r'").replace(u" BR'", u" R'")
+
+s1 = "abc\x11"
+s2 = r"abc\x11"
+s3 = R"abc\x11"
+s4 = b"abc\x11"
+s5 = B"abc\x11"
+s6 = br"abc\x11"
+s7 = Br"abc\x11"
+s8 = bR"abc\x11"
+s9 = BR"abc\x11"
+
+u1 = u"abc\x11"
+u2 = U"abc\x11"
+u3 = ur"abc\x11"
+u4 = Ur"abc\x11"
+u5 = uR"abc\x11"
+u6 = UR"abc\x11"

tests/run/subop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f()
     (-1, -1)
     >>> p()

tests/broken/tandemstats.pyx → tests/run/tandemstats.pyx

+__doc__ = u"""
+>>> result() == (99, 17*42, 17*42)
+True
+"""
+
 cdef int i, j, k
 i = 17; j = 42; k = i * j
 if j > k: i = 88
 else: i = 99; j = k
+
+def result():
+    return (i,j,k)

tests/run/tuple.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1,2,3,4,5)
     ()
     >>> g(1,2,3,4,5)

tests/run/tuplereassign.pyx

-__doc__ = """
+__doc__ = u"""
     >>> test1( (1,2,3) )
     1
     >>> test3( (1,2,3) )

tests/run/unicodefunction.pyx

-__doc__ = """
+__doc__ = u"""
    >>> u('test')
    u'test'
    >>> z
@@ -16,8 +16,12 @@ __doc__ = """
 #   u'testing a C subtype'
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+
 u = unicode
-z = unicode('test')
+z = unicode(u'test')
 
 def c(string):
     return unicode(string)

tests/run/unicodeliterals.pyx

@@ -2,7 +2,7 @@
 
 __doc__ = r"""
     >>> sa
-    'abc'
+    b'abc'
     >>> ua
     u'abc'
     >>> b
@@ -19,7 +19,7 @@ __doc__ = r"""
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
     u'\x00'
-""" + """
+""".decode(u"ASCII") + """
     >>> len(sa)
     3
     >>> len(ua)
@@ -38,9 +38,7 @@ __doc__ = r"""
     12
     >>> len(null)
     1
-""" + u"""
-    >>> sa == 'abc'
-    True
+""".decode(u"ASCII") + u"""
     >>> ua == u'abc'
     True
     >>> b == u'123'
@@ -63,6 +61,12 @@ __doc__ = r"""
     True
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+else:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 sa = 'abc'
 ua = u'abc'
 
@@ -72,5 +76,5 @@ d = u'üÖä'
 e = u'\x03\x67\xf8\uf8d2Søk ik'
 f = u'\xf8'
 
-add = u'Søk ik' + u'üÖä' + 'abc'
+add = u'Søk ik' + u'üÖä' + u'abc'
 null = u'\x00'

tests/run/unicodeliteralsdefault.pyx

@@ -8,7 +8,7 @@
 
 __doc__ = r"""
     >>> sa
-    'abc'
+    b'abc'
     >>> ua
     u'abc'
     >>> b
@@ -25,7 +25,7 @@ __doc__ = r"""
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
     u'\x00'
-""" + """
+""".decode(u"ASCII") + """
     >>> len(sa)
     3
     >>> len(ua)
@@ -44,9 +44,7 @@ __doc__ = r"""
     12
     >>> len(null)
     1
-""" + u"""
-    >>> sa == 'abc'
-    True
+""".decode(u"ASCII") + u"""
     >>> ua == u'abc'
     True
     >>> b == u'123'
@@ -69,6 +67,12 @@ __doc__ = r"""
     True
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+else:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 sa = 'abc'
 ua = u'abc'
 
@@ -78,5 +82,5 @@ d = u'üÖä'
 e = u'\x03\x67\xf8\uf8d2Søk ik'
 f = u'\xf8'
 
-add = u'Søk ik' + u'üÖä' + 'abc'
+add = u'Søk ik' + u'üÖä' + u'abc'
 null = u'\x00'

tests/run/unicodeliteralslatin1.pyx

@@ -2,7 +2,7 @@
 
 __doc__ = r"""
     >>> sa
-    'abc'
+    b'abc'
     >>> ua
     u'abc'
     >>> b
@@ -19,7 +19,7 @@ __doc__ = r"""
     u'S\xf8k ik\xfc\xd6\xe4abc'
     >>> null
     u'\x00'
-""" + """
+""".decode(u"ASCII") + """
     >>> len(sa)
     3
     >>> len(ua)
@@ -38,9 +38,7 @@ __doc__ = r"""
     12
     >>> len(null)
     1
-""" + u"""
-    >>> sa == 'abc'
-    True
+""".decode(u"ASCII") + u"""
     >>> ua == u'abc'
     True
     >>> b == u'123'
@@ -63,6 +61,12 @@ __doc__ = r"""
     True
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u" u'", u" '")
+else:
+    __doc__ = __doc__.replace(u" b'", u" '")
+
 sa = 'abc'
 ua = u'abc'
 
@@ -72,5 +76,5 @@ d = u'
 e = u'\x03\x67\xf8\uf8d2Sk ik'
 f = u'\xf8'
 
-add = u'Sk ik' + u'' + 'abc'
+add = u'Sk ik' + u'' + u'abc'
 null = u'\x00'

tests/run/unop.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1, 2, 3)
     (-3, -4, 1)
 """

tests/run/unpack.pyx

-__doc__ = """
+__doc__ = u"""
     >>> f(1, (2,), (3,4,5), (6,(7,(8,9))), 0)
     (8, 9, (8, 9), (6, (7, (8, 9))), 0)
 """

tests/run/unpacklistcomp.pyx

-__doc__ = """
+__doc__ = u"""
     >>> unpack_normal([1,2])
     (1, 2)
     >>> unpack_normal([1,2,3]) # doctest: +ELLIPSIS

tests/run/varargcall.pyx

-__doc__ = """
+__doc__ = u"""
     >>> swallow()
 """
 

tests/broken/watts1.pyx → tests/run/watts1.pyx

+__doc__ = u"""
+>>> test() == 55 + 66
+True
+"""
+
+
 def test():
     cdef int a,b
     foo=(55,66)
     a,b=foo
+    return a + b

tests/compile/withnogil.pyx → tests/run/withnogil.pyx

-cdef object f(object x):
+__doc__ = u"""
+    >>> f(1)
+    (1, 17)
+    >>> g()
+    1
+"""
+
+def f(x):
     cdef int y
-    #z = 42
+    z = 42
     with nogil:
-        pass#y = 17
-    #z = 88
+        y = 17
+    z = x
+    return z,y
 
-cdef object g():
+def g():
     with nogil:
         h()
+    return 1
 
 cdef int h() except -1:
     pass

tests/run/wundram1.pyx

-__doc__ = """
+__doc__ = u"""
     >>> x
     5L
 """
 
+import sys
+if sys.version_info[0] >= 3:
+    __doc__ = __doc__.replace(u"5L", u"5")
+
 cdef unsigned int ui
 ui = 5
 x = ui