merge

Cython/Compiler/Buffer.py

@@ -7,7 +7,12 @@ from Cython.Utils import EncodedString
 from Cython.Compiler.Errors import CompileError
 import Interpreter
 import PyrexTypes
-from sets import Set as set
+
+try:
+    set
+except NameError:
+    from sets import Set as set
+
 import textwrap
 
 # Code cleanup ideas:

Cython/Compiler/CmdLine.py

@@ -12,7 +12,7 @@ Cython language.  Cython is based on Pyrex by Greg Ewing.
 Usage: cython [options] sourcefile.pyx ...
 
 Options:
-  -v, --version                  Display version number of cython compiler
+  -V, --version                  Display version number of cython compiler
   -l, --create-listing           Write error messages to a listing file
   -I, --include-dir <directory>  Search for include files in named directory
                                  (multiply include directories are allowed).
@@ -21,6 +21,7 @@ Options:
   -t, --timestamps               Only compile newer source files (implied with -r)
   -f, --force                    Compile all source files (overrides implied -t)
   -q, --quiet                    Don't print module names in recursive mode
+  -v, --verbose                  Be verbose, print file names on multiple compilation
   -p, --embed-positions          If specified, the positions in Cython files of each
                                  function definition is embedded in its docstring.
   -z, --pre-import <module>      If specified, assume undeclared names in this 
@@ -34,7 +35,7 @@ Options:
                                  are searched from)
 
   -D, --no-docstrings            Remove docstrings.
-  -a, --annotate                 Produce an colorized version of the source.
+  -a, --annotate                 Produce a colorized HTML version of the source.
   --convert-range                Convert for loops using range() function to for...from loops. 
   --cplus                        Output a c++ rather than c file.
   -O, --option <name>=<value>[,<name=value,...] Overrides an optimization/code generation option
@@ -72,7 +73,7 @@ def parse_command_line(args):
     while args:
         if args[0].startswith("-"):
             option = pop_arg()
-            if option in ("-v", "--version"):
+            if option in ("-V", "--version"):
                 options.show_version = 1
             elif option in ("-l", "--create-listing"):
                 options.use_listing_file = 1
@@ -97,6 +98,8 @@ def parse_command_line(args):
                 options.timestamps = 1
             elif option in ("-f", "--force"):
                 options.timestamps = 0
+            elif option in ("-v", "--verbose"):
+                options.verbose += 1
             elif option in ("-p", "--embed-positions"):
                 Options.embed_pos_in_docstring = 1
             elif option in ("-z", "--pre-import"):

Cython/Compiler/Code.py

@@ -10,7 +10,10 @@ from PyrexTypes import py_object_type, typecast
 from TypeSlots import method_coexist
 from Scanning import SourceDescriptor
 from Cython.StringIOTree import StringIOTree
-from sets import Set as set
+try:
+    set
+except NameError:
+    from sets import Set as set
 
 class FunctionState(object):
     # return_label     string          function return point label
@@ -515,11 +518,18 @@ class CCodeWriter(object):
         self.marker = None
         return self
 
+    def put_safe(self, code):
+        # put code, but ignore {}
+        self.write(code)
+        self.bol = 0
+
     def put(self, code):
+        fix_indent = False
         dl = code.count("{") - code.count("}")
         if dl < 0:
             self.level += dl
         elif dl == 0 and code.startswith('}'):
+            fix_indent = True
             self.level -= 1
         if self.bol:
             self.indent()
@@ -527,7 +537,7 @@ class CCodeWriter(object):
         self.bol = 0
         if dl > 0:
             self.level += dl
-        elif dl == 0 and code.startswith('}'):
+        elif fix_indent:
             self.level += 1
         return self
 
@@ -616,7 +626,7 @@ class CCodeWriter(object):
         self.put(entry.type.declaration_code(entry.cname,
             dll_linkage = dll_linkage))
         if entry.init is not None:
-            self.put(" = %s" % entry.type.literal_code(entry.init))
+            self.put_safe(" = %s" % entry.type.literal_code(entry.init))
         self.putln(";")
 
     def put_temp_declarations(self, func_context):

Cython/Compiler/ExprNodes.py

@@ -14,6 +14,7 @@ from Builtin import list_type, tuple_type, dict_type
 import Symtab
 import Options
 from Annotate import AnnotationItem
+from Cython import Utils
 
 from Cython.Debugging import print_call_chain
 from DebugFlags import debug_disposal_code, debug_temp_alloc, \
@@ -639,10 +640,10 @@ class CharNode(ConstNode):
     type = PyrexTypes.c_char_type
     
     def compile_time_value(self, denv):
-        return ord(self.value)
+        return ord(self.value.byteencode())
     
     def calculate_result_code(self):
-        return "'%s'" % self.value
+        return "'%s'" % Utils.escape_character(self.value.byteencode())
 
 
 class IntNode(ConstNode):
@@ -707,8 +708,7 @@ class StringNode(ConstNode):
     def coerce_to(self, dst_type, env):
         if dst_type.is_int:
             if not self.type.is_pyobject and len(self.entry.init) == 1:
-                # we use the *encoded* value here
-                return CharNode(self.pos, value=self.entry.init)
+                return CharNode(self.pos, value=self.value)
             else:
                 error(self.pos, "Only coerce single-character ascii strings can be used as ints.")
                 return self
@@ -737,6 +737,32 @@ class StringNode(ConstNode):
             return self.entry.cname
 
 
+class UnicodeNode(PyConstNode):
+    #  entry   Symtab.Entry
+
+    type = PyrexTypes.c_unicode_type
+
+    def analyse_types(self, env):
+        self.entry = env.add_string_const(self.value)
+        env.add_py_string(self.entry)
+
+    def calculate_result_code(self):
+        return self.entry.pystring_cname
+    
+    def _coerce_to(self, dst_type, env):
+        if not dst_type.is_pyobject:
+            node = StringNode(self.pos, entry = entry, type = py_object_type)
+            return ConstNode.coerce_to(node, dst_type, env)
+        else:
+            return self
+        # We still need to perform normal coerce_to processing on the
+        # result, because we might be coercing to an extension type,
+        # in which case a type test node will be needed.
+        
+    def compile_time_value(self, env):
+        return self.value
+
+
 class IdentifierStringNode(ConstNode):
     # A Python string that behaves like an identifier, e.g. for
     # keyword arguments in a call, or for imported names
@@ -870,8 +896,6 @@ class NameNode(AtomicExprNode):
         env.control_flow.set_state(self.pos, (self.name, 'source'), 'assignment')
         if self.entry.is_declared_generic:
             self.result_ctype = py_object_type
-        if self.entry.is_pyglobal and self.entry.is_member:
-            env.use_utility_code(type_cache_invalidation_code)
     
     def analyse_types(self, env):
         if self.entry is None:
@@ -1025,7 +1049,7 @@ class NameNode(AtomicExprNode):
                         self.interned_cname,
                         rhs.py_result()))
                 # in Py2.6+, we need to invalidate the method cache
-                code.putln("__Pyx_TypeModified(%s);" %
+                code.putln("PyType_Modified(%s);" %
                            entry.scope.parent_type.typeptr_cname)
             else: 
                 code.put_error_if_neg(self.pos,
@@ -1348,6 +1372,9 @@ class IndexNode(ExprNode):
         self.is_buffer_access = False
 
         self.base.analyse_types(env)
+        # Handle the case where base is a literal char* (and we expect a string, not an int)
+        if isinstance(self.base, StringNode):
+            self.base = self.base.coerce_to_pyobject(env)
 
         skip_child_analysis = False
         buffer_access = False
@@ -4433,43 +4460,6 @@ static INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) {
 
 #------------------------------------------------------------------------------------
 
-type_cache_invalidation_code = [
-"""
-#if PY_VERSION_HEX >= 0x02060000
-/* #define __Pyx_TypeModified(t) PyType_Modified(t) */  /* Py3.0beta1 */
-static void __Pyx_TypeModified(PyTypeObject* type); /*proto*/
-#else
-  #define __Pyx_TypeModified(t)
-#endif
-""","""
-#if PY_VERSION_HEX >= 0x02060000
-/* copied from typeobject.c in Python 3.0a5 */
-static void __Pyx_TypeModified(PyTypeObject* type) {
-    PyObject *raw, *ref;
-    Py_ssize_t i, n;
-
-    if (!PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG))
-        return;
-
-    raw = type->tp_subclasses;
-    if (raw != NULL) {
-        n = PyList_GET_SIZE(raw);
-        for (i = 0; i < n; i++) {
-            ref = PyList_GET_ITEM(raw, i);
-            ref = PyWeakref_GET_OBJECT(ref);
-            if (ref != Py_None) {
-                __Pyx_TypeModified((PyTypeObject *)ref);
-            }
-        }
-    }
-    type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG;
-}
-#endif
-"""
-]
-
-#------------------------------------------------------------------------------------
-
 # If the is_unsigned flag is set, we need to do some extra work to make 
 # sure the index doesn't become negative. 
 

Cython/Compiler/Lexicon.py

@@ -63,8 +63,9 @@ def make_lexicon():
     three_oct = octdigit + octdigit + octdigit
     two_hex = hexdigit + hexdigit
     four_hex = two_hex + two_hex
-    escapeseq = Str("\\") + (two_oct | three_oct | two_hex |
-                             Str('u') + four_hex | Str('x') + two_hex | AnyChar)
+    escapeseq = Str("\\") + (two_oct | three_oct |
+                             Str('u') + four_hex | Str('x') + two_hex |
+                             Str('U') + four_hex + four_hex | AnyChar)
     
 
     deco = Str("@")

Cython/Compiler/Main.py

@@ -189,7 +189,9 @@ class Context:
         scope = None
         pxd_pathname = None
         if not module_name_pattern.match(module_name):
-            raise CompileError((path, 0, 0),
+            if pos is None:
+                pos = (module_name, 0, 0)
+            raise CompileError(pos,
                 "'%s' is not a valid module name" % module_name)
         if "." not in module_name and relative_to:
             if debug_find_module:
@@ -477,13 +479,6 @@ def create_default_resultobj(compilation_source, options):
         else:
             c_suffix = ".c"
         result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
-    # The below doesn't make any sense? Why is it there?
-    c_stat = None
-    if result.c_file:
-        try:
-            c_stat = os.stat(result.c_file)
-        except EnvironmentError:
-            pass
     return result
 
 def run_pipeline(source, options, full_module_name = None):
@@ -615,8 +610,6 @@ def compile_single(source, options, full_module_name = None):
     recursion.
     """
     return run_pipeline(source, options, full_module_name)
-#    context = Context(options.include_path)
-#    return context.compile(source, options, full_module_name)
 
 
 def compile_multiple(sources, options):
@@ -636,15 +629,14 @@ def compile_multiple(sources, options):
         timestamps = recursive
     verbose = options.verbose or ((recursive or timestamps) and not options.quiet)
     for source in sources:
-        context = Context(options.include_path) # to be removed later
         if source not in processed:
+            # Compiling multiple sources in one context doesn't quite
+            # work properly yet.
             if not timestamps or context.c_file_out_of_date(source):
                 if verbose:
                     sys.stderr.write("Compiling %s\n" % source)
 
-                result = context.compile(source, options)
-                # Compiling multiple sources in one context doesn't quite
-                # work properly yet.
+                result = run_pipeline(source, options)
                 results.add(source, result)
             processed.add(source)
             if recursive:
@@ -674,10 +666,7 @@ def compile(source, options = None, c_compile = 0, c_link = 0,
             and not options.recursive:
         return compile_single(source, options, full_module_name)
     else:
-        # Hack it for wednesday dev1
-        assert len(source) == 1
-        return compile_single(source[0], options)
-#        return compile_multiple(source, options)
+        return compile_multiple(source, options)
 
 #------------------------------------------------------------------------
 #

Cython/Compiler/ModuleNode.py

@@ -23,7 +23,7 @@ import Version
 
 from Errors import error, warning
 from PyrexTypes import py_object_type
-from Cython.Utils import open_new_file, replace_suffix
+from Cython.Utils import open_new_file, replace_suffix, escape_byte_string, EncodedString
 
 
 def check_c_classes(module_node):
@@ -45,9 +45,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     
     def analyse_declarations(self, env):
         if Options.embed_pos_in_docstring:
-            env.doc = 'File: %s (starting at line %s)'%Nodes.relative_position(self.pos)
+            env.doc = EncodedString(u'File: %s (starting at line %s)' % Nodes.relative_position(self.pos))
             if not self.doc is None:
-                env.doc = env.doc + '\\n' + self.doc
+                env.doc = EncodedString(env.doc + u'\\n' + self.doc)
+                env.doc.encoding = self.doc.encoding
         else:
             env.doc = self.doc
         self.body.analyse_declarations(env)
@@ -423,6 +424,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln("  #define Py_SIZE(ob)   ((PyVarObject*)(ob))->ob_size)")
         code.putln("  #define PyVarObject_HEAD_INIT(type, size) \\")
         code.putln("          PyObject_HEAD_INIT(type) size,")
+        code.putln("  #define PyType_Modified(t)")
         code.putln("")
         code.putln("  typedef struct {")
         code.putln("     void *buf;")
@@ -513,7 +515,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln('static const char **%s;' % Naming.filetable_cname)
         if env.doc:
             code.putln('')
-            code.putln('static char %s[] = "%s";' % (env.doc_cname, env.doc))
+            code.putln('static char %s[] = "%s";' % (
+                    env.doc_cname, escape_byte_string(env.doc.utf8encode())))
     
     def generate_extern_c_macro_definition(self, code):
         name = Naming.extern_c_macro

Cython/Compiler/Nodes.py

@@ -12,7 +12,8 @@ import TypeSlots
 from PyrexTypes import py_object_type, error_type, CTypedefType, CFuncType
 from Symtab import ModuleScope, LocalScope, GeneratorLocalScope, \
     StructOrUnionScope, PyClassScope, CClassScope
-from Cython.Utils import open_new_file, replace_suffix, EncodedString
+from Cython.Utils import open_new_file, replace_suffix
+from Cython.Utils import EncodedString, escape_byte_string
 import Options
 import ControlFlow
 
@@ -762,7 +763,7 @@ class CEnumDefItemNode(StatNode):
         else:
             value = self.name
         entry = env.declare_const(self.name, enum_entry.type, 
-            value, self.pos, cname = self.cname)
+            value, self.pos, cname = self.cname, visibility = enum_entry.visibility)
         enum_entry.enum_values.append(entry)
 
 
@@ -945,6 +946,8 @@ class FuncDefNode(StatNode, BlockNode):
             if err_val is not None or exc_check:
                 code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name)
             else:
+                warning(self.entry.pos, "Unraisable exception in function '%s'." \
+                            % self.entry.qualified_name, 0)
                 code.putln(
                     '__Pyx_WriteUnraisable("%s");' % 
                         self.entry.qualified_name)
@@ -1516,7 +1519,7 @@ class DefNode(FuncDefNode):
             code.putln(
                 'static char %s[] = "%s";' % (
                     self.entry.doc_cname,
-                    self.entry.doc))
+                    escape_byte_string(self.entry.doc.utf8encode())))
         if with_pymethdef:
             code.put(
                 "static PyMethodDef %s = " % 
@@ -4281,11 +4284,12 @@ static int __Pyx_CheckKeywordStrings(
     }
     if (unlikely(!kw_allowed) && unlikely(key)) {
         PyErr_Format(PyExc_TypeError,
-                     "'%s' is an invalid keyword argument for this function",
         #if PY_MAJOR_VERSION < 3
+                     "'%s' is an invalid keyword argument for this function",
                      PyString_AsString(key));
         #else
-                     PyUnicode_AsString(key));
+                     "'%U' is an invalid keyword argument for this function",
+                     key);
         #endif
         return 0;
     }
@@ -4456,9 +4460,9 @@ static void __Pyx_AddTraceback(const char *funcname) {
     if (!py_srcfile) goto bad;
     if (%(CLINENO)s) {
         #if PY_MAJOR_VERSION < 3
-        py_funcname = PyString_FromFormat( "%%s (%%s:%%u)", funcname, %(CFILENAME)s, %(CLINENO)s);
+        py_funcname = PyString_FromFormat( "%%s (%%s:%%d)", funcname, %(CFILENAME)s, %(CLINENO)s);
         #else
-        py_funcname = PyUnicode_FromFormat( "%%s (%%s:%%u)", funcname, %(CFILENAME)s, %(CLINENO)s);
+        py_funcname = PyUnicode_FromFormat( "%%s (%%s:%%d)", funcname, %(CFILENAME)s, %(CLINENO)s);
         #endif
     }
     else {

Cython/Compiler/Optimize.py

@@ -22,7 +22,7 @@ class SwitchTransform(Visitor.VisitorTransform):
     """
     This transformation tries to turn long if statements into C switch statements. 
     The requirement is that every clause be an (or of) var == value, where the var
-    is common among all clauses and both var and value are not Python objects. 
+    is common among all clauses and both var and value are ints. 
     """
     def extract_conditions(self, cond):
     
@@ -66,6 +66,8 @@ class SwitchTransform(Visitor.VisitorTransform):
                 return node
             elif common_var is not None and not is_common_value(var, common_var):
                 return node
+            elif not var.type.is_int or sum([not cond.type.is_int for cond in conditions]):
+                return node
             else:
                 common_var = var
                 cases.append(Nodes.SwitchCaseNode(pos = if_clause.pos,

Cython/Compiler/ParseTreeTransforms.py

@@ -5,7 +5,10 @@ from Cython.Compiler.ExprNodes import *
 from Cython.Compiler.TreeFragment import TreeFragment
 from Cython.Utils import EncodedString
 from Cython.Compiler.Errors import CompileError
-from sets import Set as set
+try:
+    set
+except NameError:
+    from sets import Set as set
 import copy
 
 class NormalizeTree(CythonTransform):
@@ -356,7 +359,7 @@ class ResolveOptions(CythonTransform):
             for option in options:
                 name, value = option
                 optdict[name] = value
-            return self.visit_with_options(node, options)
+            return self.visit_with_options(node, optdict)
         else:
             return self.visit_Node(node)
                                    

Cython/Compiler/Parsing.py

@@ -2,7 +2,9 @@
 #   Pyrex Parser
 #
 
-import os, re
+import os
+import re
+import sys
 from types import ListType, TupleType
 from Scanning import PyrexScanner, FileSourceDescriptor
 import Nodes
@@ -491,6 +493,8 @@ def p_atom(s):
         kind, value = p_cat_string_literal(s)
         if kind == 'c':
             return ExprNodes.CharNode(pos, value = value)
+        elif kind == 'u':
+            return ExprNodes.UnicodeNode(pos, value = value)
         else:
             return ExprNodes.StringNode(pos, value = value)
     elif sy == 'IDENT':
@@ -584,43 +588,43 @@ def p_string_literal(s):
         sy = s.sy
         #print "p_string_literal: sy =", sy, repr(s.systring) ###
         if sy == 'CHARS':
-            systr = s.systring
-            if len(systr) == 1 and systr in "'\"\n":
-                chars.append('\\')
-            chars.append(systr)
+            chars.append(s.systring)
         elif sy == 'ESCAPE':
             systr = s.systring
             if is_raw:
                 if systr == '\\\n':
-                    chars.append(r'\\\n')
-                elif systr == r'\"':
-                    chars.append(r'\\\"')
-                elif systr == r'\\':
-                    chars.append(r'\\\\')
+                    chars.append('\\\n')
+                elif systr == '\\\"':
+                    chars.append('"')
+                elif systr == '\\\'':
+                    chars.append("'")
                 else:
-                    chars.append('\\' + systr)
+                    chars.append(systr)
             else:
                 c = systr[1]
-                if c in "'\"\\abfnrtv01234567":
-                    chars.append(systr)
+                if c in "01234567":
+                    chars.append(chr(int(systr[1:], 8)))
+                elif c in "'\"\\":
+                    chars.append(c)
+                elif c in "abfnrtv":
+                    chars.append(Utils.char_from_escape_sequence(systr))
                 elif c == '\n':
                     pass
-                elif c in 'ux':
-                    if kind == 'u':
-                        try:
-                            chars.append(
-                                systr.encode("ASCII").decode('unicode_escape'))
-                        except UnicodeDecodeError:
+                elif c in 'Uux':
+                    if kind == 'u' or c == 'x':
+                        chrval = int(systr[2:], 16)
+                        if chrval > 1114111: # sys.maxunicode:
                             s.error("Invalid unicode escape '%s'" % systr,
                                     pos = pos)
-                    elif c == 'x':
-                        chars.append('\\x0' + systr[2:])
+                        strval = unichr(chrval)
                     else:
-                        chars.append(systr)
+                        # unicode escapes in plain byte strings are not unescaped
+                        strval = systr
+                    chars.append(strval)
                 else:
-                    chars.append(r'\\' + systr[1:])
+                    chars.append('\\' + systr[1:])
         elif sy == 'NEWLINE':
-            chars.append(r'\n')
+            chars.append('\n')
         elif sy == 'END_STRING':
             break
         elif sy == 'EOF':
@@ -629,8 +633,11 @@ def p_string_literal(s):
             s.error(
                 "Unexpected token %r:%r in string literal" %
                     (sy, s.systring))
+    string = u''.join(chars)
+    if kind == 'c' and len(string) != 1:
+        error(pos, u"invalid character literal: %r" % string)
     s.next()
-    value = Utils.EncodedString( u''.join(chars) )
+    value = Utils.EncodedString(string)
     if kind != 'u':
         value.encoding = s.source_encoding
     #print "p_string_literal: value =", repr(value) ###
@@ -1630,7 +1637,7 @@ def p_buffer_access(s, base_type_node):
     # s.sy == '['
     pos = s.position()
     s.next()
-    if s.sy == ']':
+    if s.sy == ']' or s.sy == 'INT':
         # not buffer, could be [] on C type nameless array arguments
         s.put_back('[', '[')
         return base_type_node
@@ -1881,7 +1888,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0)
         if 'pxd' in s.level:
             if s.sy not in ['*', '?']:
                 error(pos, "default values cannot be specified in pxd files, use ? or *")
-            default = 1
+            default = ExprNodes.BoolNode(1)
             s.next()
         else:
             default = p_simple_expr(s)

Cython/Compiler/PyrexTypes.py

@@ -292,6 +292,13 @@ class BuiltinObjectType(PyObjectType):
     def type_test_code(self, arg):
         return 'likely(Py%s_CheckExact(%s)) || (%s) == Py_None || (PyErr_Format(PyExc_TypeError, "Expected %s, got %%s", Py_TYPE(%s)->tp_name), 0)' % (self.name[0].upper() + self.name[1:], arg, arg, self.name, arg)
 
+    def declaration_code(self, entity_code, 
+            for_display = 0, dll_linkage = None, pyrex = 0):
+        if pyrex or for_display:
+            return self.base_declaration_code(self.name, entity_code)
+        else:
+            return "%s *%s" % (public_decl("PyObject", dll_linkage), entity_code)
+
 
 class PyExtensionType(PyObjectType):
     #
@@ -996,20 +1003,6 @@ class CStringType:
         return '"%s"' % Utils.escape_byte_string(value)
 
 
-class CUTF8StringType:
-    #  Mixin class for C unicode types.
-
-    is_string = 1
-    is_unicode = 1
-    
-    to_py_function = "PyUnicode_DecodeUTF8"
-    exception_value = "NULL"
-
-    def literal_code(self, value):
-        assert isinstance(value, str)
-        return '"%s"' % Utils.escape_byte_string(value)
-
-
 class CCharArrayType(CStringType, CArrayType):
     #  C 'char []' type.
     
@@ -1018,16 +1011,6 @@ class CCharArrayType(CStringType, CArrayType):
     
     def __init__(self, size):
         CArrayType.__init__(self, c_char_type, size)
-
-
-class CUTF8CharArrayType(CUTF8StringType, CArrayType):
-    #  C 'char []' type.
-    
-    parsetuple_format = "s"
-    pymemberdef_typecode = "T_STRING_INPLACE"
-    
-    def __init__(self, size):
-        CArrayType.__init__(self, c_char_type, size)
     
 
 class CCharPtrType(CStringType, CPtrType):
@@ -1040,6 +1023,29 @@ class CCharPtrType(CStringType, CPtrType):
         CPtrType.__init__(self, c_char_type)
 
 
+class UnicodeType(BuiltinObjectType):
+    #  The Python unicode type.
+
+    is_string = 1
+    is_unicode = 1
+    
+    parsetuple_format = "U"
+
+    def __init__(self):
+        BuiltinObjectType.__init__(self, "unicode", "PyUnicodeObject")
+
+    def literal_code(self, value):
+        assert isinstance(value, str)
+        return '"%s"' % Utils.escape_byte_string(value)
+
+    def declaration_code(self, entity_code, 
+            for_display = 0, dll_linkage = None, pyrex = 0):
+        if pyrex or for_display:
+            return self.base_declaration_code(self.name, entity_code)
+        else:
+            return "%s %s[]" % (public_decl("char", dll_linkage), entity_code)
+
+
 class ErrorType(PyrexType):
     # Used to prevent propagation of error messages.
     
@@ -1105,7 +1111,7 @@ c_longdouble_type =  CFloatType(8, typestring="g")
 
 c_null_ptr_type =     CNullPtrType(c_void_type)
 c_char_array_type =   CCharArrayType(None)
-c_utf8_char_array_type =   CUTF8CharArrayType(None)
+c_unicode_type =      UnicodeType()
 c_char_ptr_type =     CCharPtrType()
 c_char_ptr_ptr_type = CPtrType(c_char_ptr_type)
 c_py_ssize_t_ptr_type =  CPtrType(c_py_ssize_t_type)

Cython/Compiler/Scanning.py

@@ -38,7 +38,10 @@ def hash_source_file(path):
     # Try to calculate a hash code for the given source file.
     # Returns an empty string if the file cannot be accessed.
     #print "Hashing", path ###
-    import md5
+    try:
+        from hashlib import md5 as new_md5
+    except ImportError:
+        from md5 import new as new_md5
     try:
         try:
             f = open(path, "rU")
@@ -54,7 +57,7 @@ def hash_source_file(path):
     # tabs by a single space.
     import re
     text = re.sub("[ \t]+", " ", text)
-    hash = md5.new(text).hexdigest()
+    hash = new_md5(text).hexdigest()
     return hash
 
 def open_pickled_lexicon(expected_hash):

Cython/Compiler/Symtab.py

@@ -15,7 +15,10 @@ from TypeSlots import \
     get_special_method_signature, get_property_accessor_signature
 import ControlFlow
 import __builtin__
-from sets import Set as set
+try:
+    set
+except NameError:
+    from sets import Set as set
 
 possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
@@ -287,14 +290,14 @@ class Scope:
     def qualify_name(self, name):
         return "%s.%s" % (self.qualified_name, name)
     
-    def declare_const(self, name, type, value, pos, cname = None):
+    def declare_const(self, name, type, value, pos, cname = None, visibility = 'private'):
         # Add an entry for a named constant.
         if not cname:
             if self.in_cinclude:
                 cname = name
             else:
                 cname = self.mangle(Naming.enum_prefix, name)
-        entry = self.declare(name, cname, type, pos, 'private')
+        entry = self.declare(name, cname, type, pos, visibility)
         entry.is_const = 1
         entry.value = value
         return entry
@@ -502,7 +505,7 @@ class Scope:
         else:
             cname = self.new_const_cname()
         if value.is_unicode:
-            c_type = PyrexTypes.c_utf8_char_array_type
+            c_type = PyrexTypes.c_unicode_type
             value = value.utf8encode()
         else:
             c_type = PyrexTypes.c_char_array_type

Cython/Compiler/Tests/TestBuffer.py

@@ -26,10 +26,6 @@ class TestBufferParsing(CythonTest):
 #        print bufnode.dump()
         # should put more here...
         
-    def test_type_fail(self):
-        self.not_parseable("Expected: type",
-                           u"cdef object[2] x")
-    
     def test_type_pos(self):
         self.parse(u"cdef object[short unsigned int, 3] x")
 

Cython/Compiler/Visitor.py

@@ -28,6 +28,10 @@ class BasicVisitor(object):
                 if m is not None:
                     break
             else:
+                print type(self), type(obj)
+                print self.access_path
+                print self.access_path[-1][0].pos
+                print self.access_path[-1][0].__dict__
                 raise RuntimeError("Visitor does not accept object: %s" % obj)
             self.dispatch_table[mname] = m
         return m(obj)

Cython/Includes/python.pxd

@@ -118,32 +118,28 @@
 # just to be sure you understand what is going on.
 #
 #################################################################
-cdef extern from "Python.h":
-    ctypedef void PyObject
-    ctypedef void PyTypeObject
-    ctypedef struct FILE
 
 
-include 'python_ref.pxi'
-include 'python_exc.pxi'
-include 'python_module.pxi'
-include 'python_mem.pxi'
-include 'python_tuple.pxi'
-include 'python_list.pxi'
-include 'python_object.pxi'
-include 'python_sequence.pxi'
-include 'python_mapping.pxi'
-include 'python_iterator.pxi'
-include 'python_type.pxi'
-include 'python_number.pxi'
-include 'python_int.pxi'
-include 'python_bool.pxi'
-include 'python_long.pxi'
-include 'python_float.pxi'
-include 'python_complex.pxi'
-include 'python_string.pxi'
-include 'python_dict.pxi'
-include 'python_instance.pxi'
-include 'python_function.pxi'
-include 'python_method.pxi'
-include 'python_set.pxi'
+from python_ref cimport *
+from python_exc cimport *
+from python_module cimport *
+from python_mem cimport *
+from python_tuple cimport *
+from python_list cimport *
+from python_object cimport *
+from python_sequence cimport *
+from python_mapping cimport *
+from python_iterator cimport *
+from python_type cimport *
+from python_number cimport *
+from python_int cimport *
+from python_bool cimport *
+from python_long cimport *
+from python_float cimport *
+from python_complex cimport *
+from python_string cimport *
+from python_dict cimport *
+from python_instance cimport *
+from python_function cimport *
+from python_method cimport *
+from python_set cimport *

Cython/Includes/python_ref.pxd

 cdef extern from "Python.h":
-    ctypedef void PyObject
     ctypedef void PyTypeObject
+    ctypedef struct PyObject:
+        Py_ssize_t ob_refcnt
+        PyTypeObject *ob_type
     ctypedef struct FILE    
 
 

Cython/StringIOTree.py

@@ -70,40 +70,21 @@ EXAMPLE:
 >>> b = a.insertion_point()
 >>> a.write('third\n')
 >>> b.write('second\n')
->>> print a.getvalue()
-first
-second
-third
-<BLANKLINE>
+>>> a.getvalue().split()
+['first', 'second', 'third']
 
 >>> c = b.insertion_point()
 >>> d = c.insertion_point()
 >>> d.write('alpha\n')
 >>> b.write('gamma\n')
 >>> c.write('beta\n')
->>> print b.getvalue()
-second
-alpha
-beta
-gamma
-<BLANKLINE>
-
+>>> b.getvalue().split()
+['second', 'alpha', 'beta', 'gamma']
 >>> i = StringIOTree()
 >>> d.insert(i)
 >>> i.write('inserted\n')
 >>> out = StringIO()
 >>> a.copyto(out)
->>> print out.getvalue()
-first
-second
-alpha
-inserted
-beta
-gamma
-third
-<BLANKLINE>
-"""
-            
-if __name__ == "__main__":
-    import doctest
-    doctest.testmod()
+>>> out.getvalue().split()
+['first', 'second', 'alpha', 'inserted', 'beta', 'gamma', 'third']
+"""
\ No newline at end of file

Cython/Utils.py

@@ -99,8 +99,53 @@ class EncodedString(unicode):
 #        return unicode.__eq__(self, other) and \
 #            getattr(other, 'encoding', '') == self.encoding
 
+char_from_escape_sequence = {
+    r'\a' : '\a',
+    r'\b' : '\b',
+    r'\f' : '\f',
+    r'\n' : '\n',
+    r'\r' : '\r',
+    r'\t' : '\t',
+    r'\v' : '\v',
+    }.get
+
+def _to_escape_sequence(s):
+    if s in '\n\r\t':
+        return repr(s)[1:-1]
+    elif s == '"':
+        return r'\"'
+    else:
+        # within a character sequence, oct passes much better than hex
+        return ''.join(['\\%03o' % ord(c) for c in s])
+
+_c_special = ('\0', '\n', '\r', '\t', '??', '"')
+_c_special_replacements = zip(_c_special, map(_to_escape_sequence, _c_special))
+
+def _build_specials_test():
+    subexps = []
+    for special in _c_special:
+        regexp = ''.join(['[%s]' % c for c in special])
+        subexps.append(regexp)
+    return re.compile('|'.join(subexps)).search
+
+_has_specials = _build_specials_test()
+
+def escape_character(c):
+    if c in '\n\r\t\\':
+        return repr(c)[1:-1]
+    elif c == "'":
+        return "\\'"
+    elif ord(c) < 32:
+        # hex works well for characters
+        return "\\x%02X" % ord(c)
+    else:
+        return c
+
 def escape_byte_string(s):
-    s = s.replace('\0', r'\x00')
+    s = s.replace('\\', '\\\\')
+    if _has_specials(s):
+        for special, replacement in _c_special_replacements:
+            s = s.replace(special, replacement)
     try:
         s.decode("ASCII")
         return s
@@ -111,7 +156,7 @@ def escape_byte_string(s):
     for c in s:
         o = ord(c)
         if o >= 128:
-            append('\\x%X' % o)
+            append('\\%3o' % o)
         else:
             append(c)
     return ''.join(l)

runtests.py

@@ -265,6 +265,7 @@ class CythonRunTestCase(CythonCompileTestCase):
         try:
             self.runCompileTest()
             if not self.cythononly:
+                sys.stderr.write('running doctests in %s ...\n' % self.module)
                 doctest.DocTestSuite(self.module).run(result)
         except Exception:
             result.addError(self, sys.exc_info())
@@ -276,7 +277,7 @@ class CythonRunTestCase(CythonCompileTestCase):
 
 class CythonUnitTestCase(CythonCompileTestCase):
     def shortDescription(self):
-        return "compiling and running unit tests in " + self.module
+        return "compiling tests in " + self.module
 
     def run(self, result=None):
         if result is None:
@@ -284,6 +285,7 @@ class CythonUnitTestCase(CythonCompileTestCase):
         result.startTest(self)
         try:
             self.runCompileTest()
+            sys.stderr.write('running tests in %s ...\n' % self.module)
             unittest.defaultTestLoader.loadTestsFromName(self.module).run(result)
         except Exception:
             result.addError(self, sys.exc_info())
@@ -380,6 +382,9 @@ if __name__ == '__main__':
     parser.add_option("--cython-only", dest="cythononly",
                       action="store_true", default=False,
                       help="only compile pyx to c, do not run C compiler or run the tests")
+    parser.add_option("--sys-pyregr", dest="system_pyregr",
+                      action="store_true", default=False,
+                      help="run the regression tests of the CPython installation")
     parser.add_option("-C", "--coverage", dest="coverage",
                       action="store_true", default=False,
                       help="collect source coverage data for the Compiler")
@@ -392,6 +397,13 @@ if __name__ == '__main__':
 
     options, cmd_args = parser.parse_args()
 
+    if sys.version_info[0] >= 3:
+        # make sure we do not import (or run) Cython itself
+        options.doctests    = False
+        options.with_cython = False
+        options.unittests   = False
+        options.pyregr      = False
+
     if options.coverage:
         import coverage
         coverage.erase()
@@ -420,11 +432,11 @@ if __name__ == '__main__':
 
     if WITH_CYTHON:
         from Cython.Compiler.Version import version
-        print("Running tests against Cython %s" % version)
+        sys.stderr.write("Running tests against Cython %s\n" % version)
     else:
-        print("Running tests without Cython.")
-    print("Python %s" % sys.version)
-    print("")
+        sys.stderr.write("Running tests without Cython.\n")
+    sys.stderr.write("Python %s\n" % sys.version)
+    sys.stderr.write("\n")
 
     import re
     selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
@@ -443,7 +455,16 @@ if __name__ == '__main__':
         filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
                                 options.annotate_source, options.cleanup_workdir,
                                 options.cleanup_sharedlibs, options.pyregr, options.cythononly)
-        test_suite.addTests([filetests.build_suite()])
+        test_suite.addTest(filetests.build_suite())
+
+    if options.system_pyregr:
+        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
+                                options.annotate_source, options.cleanup_workdir,
+                                options.cleanup_sharedlibs, True)
+        test_suite.addTest(
+            filetests.handle_directory(
+                os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test'),
+                'pyregr'))
 
     unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
 

tests/compile/arrayargs.pyx

@@ -11,3 +11,9 @@ cdef extern from *:
 
     ctypedef MyStruct* MyStructP
     cdef void baz(MyStructP[])
+
+cdef struct OtherStruct:
+    int a
+
+a = sizeof(int[23][34])
+b = sizeof(OtherStruct[43])

tests/run/bufaccess.pyx

@@ -14,7 +14,7 @@ cimport python_buffer
 cimport stdio
 cimport cython
 
-cimport refcount
+from python_ref cimport PyObject
 
 __test__ = {}
 setup_string = u"""
@@ -726,7 +726,7 @@ def decref(*args):
     for item in args: Py_DECREF(item)
 
 def get_refcount(x):
-    return refcount.CyTest_GetRefcount(x)
+    return (<PyObject*>x).ob_refcnt
 
 @testcase
 def printbuf_object(object[object] buf, shape):
@@ -749,7 +749,7 @@ def printbuf_object(object[object] buf, shape):
     """
     cdef int i
     for i in range(shape[0]):
-        print repr(buf[i]), refcount.CyTest_GetRefcount(buf[i])
+        print repr(buf[i]), (<PyObject*>buf[i]).ob_refcnt
 
 @testcase
 def assign_to_object(object[object] buf, int idx, obj):

tests/run/builtinnames.pyx

 __doc__ = u"""
->>> test_c(u'abc')
+>>> test_c('abc')
 fileabc
 typeabc
->>> print test_file_py(u'abc')
+>>> print(test_file_py('abc'))
 abc
->>> print range(u'abc')
+>>> print(range('abc'))
 rangeabc
 """
 
 
 def test_file_py(file):
+    assert isinstance(file, (str, unicode)), \
+        u"not a string, found '%s' instead" % file.__class__.__name__
     return file
 
 cdef test_file_c(file):
+    assert isinstance(file, (str, unicode)), \
+        u"not a string, found '%s' instead" % file.__class__.__name__
     return u'file' + file
 
 

tests/run/cdef_opt.pxd

+cdef class A:
+    cpdef foo(self, bint a=*, b=*)

tests/run/cdef_opt.pyx

+__doc__ = """
+    >>> a = A()
+    >>> a.foo()
+    (True, 'yo')
+    >>> a.foo(False)
+    (False, 'yo')
+    >>> a.foo(10, 'yes')
+    (True, 'yes')
+"""
+
+cdef class A:
+    cpdef foo(self, bint a=True, b="yo"):
+        return a, b

tests/run/charescape.pyx

+__doc__ = u"""
+>>> s = test()
+>>> assert s == ''.join([chr(i) for i in range(1,49)]), s
+"""
+
+def test():
+    cdef char s[50]
+
+    s[ 0] = c'\0'
+    s[ 1] = c'\x01'
+    s[ 2] = c'\x02'
+    s[ 3] = c'\x03'
+    s[ 4] = c'\x04'
+    s[ 5] = c'\x05'
+    s[ 6] = c'\x06'
+    s[ 7] = c'\x07'
+    s[ 8] = c'\x08'
+    s[ 9] = c'\x09'
+    s[10] = c'\x0A'
+    s[11] = c'\x0B'
+    s[12] = c'\x0C'
+    s[13] = c'\x0D'
+    s[14] = c'\x0E'
+    s[15] = c'\x0F'
+    s[16] = c'\x10'
+    s[17] = c'\x11'
+    s[18] = c'\x12'
+    s[19] = c'\x13'
+    s[20] = c'\x14'
+    s[21] = c'\x15'
+    s[22] = c'\x16'
+    s[23] = c'\x17'
+    s[24] = c'\x18'
+    s[25] = c'\x19'
+    s[26] = c'\x1A'
+    s[27] = c'\x1B'
+    s[28] = c'\x1C'
+    s[29] = c'\x1D'
+    s[30] = c'\x1E'
+    s[31] = c'\x1F'
+    s[32] = c'\x20'
+    s[33] = c'\x21'
+    s[34] = c'\x22'
+    s[35] = c'\x23'
+    s[36] = c'\x24'
+    s[37] = c'\x25'
+    s[38] = c'\x26'
+    s[39] = c'\x27'
+    s[40] = c'\x28'
+    s[41] = c'\x29'
+    s[42] = c'\x2A'
+    s[43] = c'\x2B'
+    s[44] = c'\x2C'
+    s[45] = c'\x2D'
+    s[46] = c'\x2E'
+    s[47] = c'\x2F'
+    s[48] = c'\x30'
+
+    s[49] = c'\x00'
+
+    assert s[ 0] == c'\x00'
+    assert s[49] == c'\0'
+
+    return &s[1]

tests/run/literalslice.pyx

+__doc__ = u"""
+  >>> test_str(1)
+  b'b'
+
+  >>> test_unicode_ascii(2)
+  u'c'
+  >>> test_unicode(2) == u'\u00e4'
+  True
+
+  >>> test_int_list(2)
+  3
+  >>> test_str_list(1)
+  b'bcd'
+
+  >>> test_int_tuple(2)
+  3
+  >>> test_str_tuple(0)
+  b'a'
+  >>> test_mix_tuple(1)
+  b'abc'
+  >>> test_mix_tuple(0)
+  1
+"""
+
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace("  b'", "  '")
+else:
+    __doc__ = __doc__.replace("  u'", "  '")
+
+
+def test_str(n):
+    return "abcd"[n]
+
+def test_unicode_ascii(n):
+    return u"abcd"[n]
+
+def test_unicode(n):
+    return u"\u00fc\u00f6\u00e4"[n]
+
+def test_int_list(n):
+    return [1,2,3,4][n]
+
+def test_str_list(n):
+    return ["a","bcd","efg","xyz"][n]
+
+def test_int_tuple(n):
+    return (1,2,3,4)[n]
+
+def test_str_tuple(n):
+    return ("a","bcd","efg","xyz")[n]
+
+def test_mix_tuple(n):
+    return (1, "abc", u"\u00fc", 1.1)[n]

tests/run/strescapes.pyx

+__doc__ = u"""
+
+>>> py_strings = [
+... b'\\x1234',
+... b'\\x0A12\\x0C34',
+... b'\\x0A57',
+... b'\\x0A',
+... b'\\'',
+... b"\\'",
+... b"\\"",
+... b'\\"',
+... b'abc\\x12def',
+... u'\\u1234',
+... u'\\U00001234',
+... b'\\u1234',
+... b'\\U00001234',
+... b'\\n\\r\\t',
+... b':>',
+... b'??>',
+... b'\\0\\0\\0',
+... ]
+
+>>> for i, (py_string, (c_string, length)) in enumerate(zip(py_strings, c_strings)):
+...     assert py_string == c_string, "%d: %r != %r" % (i, py_string, c_string)
+...     assert len(py_string) == length, (
+...         "%d: wrong length of %r, got %d, expected %d" % (
+...             i, py_string, len(py_string), length))
+...     assert len(c_string) == length, (
+...         "%d: wrong length of %r, got %d, expected %d" % (
+...             i, c_string, len(c_string), length))
+
+"""
+
+import sys
+if sys.version_info[0] < 3:
+    __doc__ = __doc__.replace(u" b'", u" '").replace(u' b"', u' "')
+else:
+    __doc__ = __doc__.replace(u" u'", u" '").replace(u' u"', u' "')
+
+c_strings = [
+(b'\x1234', 3),
+(b'\x0A12\x0C34', 6),
+(b'\x0A57', 3),
+(b'\x0A', 1),
+(b'\'', 1),
+(b"\'", 1),
+(b"\"", 1),
+(b'\"', 1),
+(b'abc\x12def', 7),
+(u'\u1234', 1),
+(u'\U00001234', 1),
+(b'\u1234', 6),
+(b'\U00001234', 10),
+(b'\n\r\t', 3),
+(b':>', 2),
+(b'??>', 3),
+(b'\0\0\0', 3),
+]

tests/run/withstat.pyx

@@ -12,12 +12,12 @@ exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
 >>> with_exception(None)
 enter
 value
-exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
+exit <type 'type'> <type 'MyException'> <type 'traceback'>
 outer except
 >>> with_exception(True)
 enter
 value
-exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
+exit <type 'type'> <type 'MyException'> <type 'traceback'>
 >>> multitarget()
 enter
 1 2 3 4 5
@@ -32,20 +32,23 @@ enter
 exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
 """
 
+def typename(t):
+    return u"<type '%s'>" % type(t).__name__
+
 class MyException(Exception):
     pass
 
-class ContextManager:
+class ContextManager(object):
     def __init__(self, value, exit_ret = None):
         self.value = value
         self.exit_ret = exit_ret
 
     def __exit__(self, a, b, tb):
-        print "exit", type(a), type(b), type(tb)
+        print u"exit", typename(a), typename(b), typename(tb)
         return self.exit_ret
         
     def __enter__(self):
-        print "enter"
+        print u"enter"
         return self.value
 
 def no_as():