big C++ mergeback

.hgtags

@@ -15,3 +15,8 @@ a4abf0156540db4d3ebaa95712b65811c43c5acb 0.11-beta
 6454db601984145f38e28d34176fca8a3a22329c 0.11.1
 af6f1bed8cd40a2edefb57d3eacbc9274a8788b4 0.11.2.rc1
 15ad532e2127840ae09dfbe46ccc80ac8c562f99 0.11.2
+eb00d00a73c13b6aa8b440fe07cd7acb52a060e8 0.11.3.rc0
+7c695fe49fd6912f52d995fe512d66baacf90ee6 0.11.3
+4208042ceeae634f5c0999b8ab75f69faf46b6db 0.12.alpha0
+e77827f09af67560aa82a18feab778f71ca0a9d3 0.12.rc0
+fae19937e4945c59a5d9d62c63f1c3b09046c3a3 0.12

Cython/CodeWriter.py

@@ -132,6 +132,7 @@ class CodeWriter(TreeVisitor):
     def visit_IntNode(self, node):
         self.put(node.value)
 
+    # FIXME: represent string nodes correctly
     def visit_StringNode(self, node):
         value = node.value
         if value.encoding is not None:

Cython/Compiler/AnalysedTreeTransforms.py

+from Cython.Compiler.Visitor import VisitorTransform, ScopeTrackingTransform, TreeVisitor
+from Nodes import StatListNode, SingleAssignmentNode, CFuncDefNode
+from ExprNodes import DictNode, DictItemNode, NameNode, UnicodeNode, NoneNode, \
+                      ExprNode, AttributeNode, ModuleRefNode, DocstringRefNode
+from PyrexTypes import py_object_type
+from Builtin import dict_type
+from StringEncoding import EncodedString
+import Naming
+
+class AutoTestDictTransform(ScopeTrackingTransform):
+    # Handles autotestdict directive
+
+    blacklist = ['__cinit__', '__dealloc__', '__richcmp__', '__nonzero__']
+
+    def visit_ModuleNode(self, node):
+        self.scope_type = 'module'
+        self.scope_node = node
+        if self.current_directives['autotestdict']:
+            assert isinstance(node.body, StatListNode)
+
+            # First see if __test__ is already created
+            if u'__test__' in node.scope.entries:
+                # Do nothing
+                return node
+            
+            pos = node.pos
+
+            self.tests = []
+            self.testspos = node.pos
+
+            test_dict_entry = node.scope.declare_var(EncodedString(u'__test__'),
+                                                     py_object_type,
+                                                     pos,
+                                                     visibility='public')
+            create_test_dict_assignment = SingleAssignmentNode(pos,
+                lhs=NameNode(pos, name=EncodedString(u'__test__'),
+                             entry=test_dict_entry),
+                rhs=DictNode(pos, key_value_pairs=self.tests))
+            self.visitchildren(node)
+            node.body.stats.append(create_test_dict_assignment)
+
+            
+        return node
+
+    def add_test(self, testpos, name, func_ref_node):
+        # func_ref_node must evaluate to the function object containing
+        # the docstring, BUT it should not be the function itself (which
+        # would lead to a new *definition* of the function)
+        pos = self.testspos
+        keystr = u'%s (line %d)' % (name, testpos[1])
+        key = UnicodeNode(pos, value=EncodedString(keystr))
+
+        value = DocstringRefNode(pos, func_ref_node)
+        self.tests.append(DictItemNode(pos, key=key, value=value))
+    
+    def visit_FuncDefNode(self, node):
+        if node.doc:
+            if isinstance(node, CFuncDefNode) and not node.py_func:
+                # skip non-cpdef cdef functions
+                return node
+            
+            pos = self.testspos
+            if self.scope_type == 'module':
+                parent = ModuleRefNode(pos)
+                name = node.entry.name
+            elif self.scope_type in ('pyclass', 'cclass'):
+                if isinstance(node, CFuncDefNode):
+                    name = node.py_func.name
+                else:
+                    name = node.name
+                if self.scope_type == 'cclass' and name in self.blacklist:
+                    return node
+                mod = ModuleRefNode(pos)
+                if self.scope_type == 'pyclass':
+                    clsname = self.scope_node.name
+                else:
+                    clsname = self.scope_node.class_name
+                parent = AttributeNode(pos, obj=mod,
+                                       attribute=clsname,
+                                       type=py_object_type,
+                                       is_py_attr=True,
+                                       is_temp=True)
+                name = "%s.%s" % (clsname, node.entry.name)
+            else:
+                assert False
+            getfunc = AttributeNode(pos, obj=parent,
+                                    attribute=node.entry.name,
+                                    type=py_object_type,
+                                    is_py_attr=True,
+                                    is_temp=True)
+            self.add_test(node.pos, name, getfunc)
+        return node
+

Cython/Compiler/Annotate.py

@@ -93,10 +93,13 @@ class AnnotationCCodeWriter(CCodeWriter):
 body { font-family: courier; font-size: 12; }
 
 .code  { font-size: 9; color: #444444; display: none; margin-left: 20px; }
-.py_api  { color: red; }
-.pyx_api  { color: #FF3000; }
-.py_macro_api  { color: #FF8000; }
-.error_goto  { color: #FF8000; }
+.py_c_api  { color: red; }
+.py_macro_api  { color: #FF7000; }
+.pyx_c_api  { color: #FF3000; }
+.pyx_macro_api  { color: #FF7000; }
+.refnanny  { color: #FFA000; }
+
+.error_goto  { color: #FFA000; }
 
 .tag  {  }
 
@@ -126,10 +129,12 @@ function toggleDiv(id) {
         f.write(u'<p>Raw output: <a href="%s">%s</a>\n' % (c_file, c_file))
         k = 0
         
-        py_c_api = re.compile(u'(Py[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]+)')
-        pyx_api = re.compile(u'(__Pyx[A-Za-z_]+)\(')
-        py_marco_api = re.compile(u'(Py[A-Za-z]*_[A-Z][A-Z_]+)')
+        py_c_api = re.compile(u'(Py[A-Z][a-z]+_[A-Z][a-z][A-Za-z_]+)\(')
+        py_marco_api = re.compile(u'(Py[A-Z][a-z]+_[A-Z][A-Z_]+)\(')
+        pyx_c_api = re.compile(u'(__Pyx_[A-Z][a-z_][A-Za-z_]+)\(')
+        pyx_macro_api = re.compile(u'(__Pyx_[A-Z][A-Z_]+)\(')
         error_goto = re.compile(ur'((; *if .*)? \{__pyx_filename = .*goto __pyx_L\w+;\})')
+        refnanny = re.compile(u'(__Pyx_X?(GOT|GIVE)REF|__Pyx_RefNanny[A-Za-z]+)')
         
         for line in lines:
 
@@ -139,14 +144,17 @@ function toggleDiv(id) {
             except KeyError:
                 code = ''
                 
-            code, c_api_calls = py_c_api.subn(ur"<span class='py_api'>\1</span>", code)
-            code, pyx_api_calls = pyx_api.subn(ur"<span class='pyx_api'>\1</span>(", code)
-            code, macro_api_calls = py_marco_api.subn(ur"<span class='py_macro_api'>\1</span>", code)
+            code, py_c_api_calls = py_c_api.subn(ur"<span class='py_c_api'>\1</span>(", code)
+            code, pyx_c_api_calls = pyx_c_api.subn(ur"<span class='pyx_c_api'>\1</span>(", code)
+            code, py_macro_api_calls = py_marco_api.subn(ur"<span class='py_macro_api'>\1</span>(", code)
+            code, pyx_macro_api_calls = pyx_macro_api.subn(ur"<span class='pyx_macro_api'>\1</span>(", code)
+            code, refnanny_calls = refnanny.subn(ur"<span class='refnanny'>\1</span>", code)
             code, error_goto_calls = error_goto.subn(ur"<span class='error_goto'>\1</span>", code)
             
             code = code.replace(u"<span class='error_goto'>;", u";<span class='error_goto'>")
             
-            color = u"FFFF%02x" % int(255/(1+(5*c_api_calls+2*pyx_api_calls+macro_api_calls)/10.0))
+            score = 5*py_c_api_calls + 2*pyx_c_api_calls + py_macro_api_calls + pyx_macro_api_calls - refnanny_calls
+            color = u"FFFF%02x" % int(255/(1+score/10.0))
             f.write(u"<pre class='line' style='background-color: #%s' onclick='toggleDiv(\"line%s\")'>" % (color, k))
 
             f.write(u" %d: " % k)

Cython/Compiler/AutoDocTransforms.py

@@ -3,9 +3,7 @@ from Cython.Compiler.Nodes import DefNode, CFuncDefNode
 from Cython.Compiler.Errors import CompileError
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler import Options
-from Cython.Compiler import PyrexTypes
-
-
+from Cython.Compiler import PyrexTypes, ExprNodes
 
 class EmbedSignature(CythonTransform):
 
@@ -16,15 +14,26 @@ class EmbedSignature(CythonTransform):
         self.class_node = None
 
     def _fmt_arg_defv(self, arg):
-        if not arg.default:
+        default_val = arg.default
+        if not default_val:
             return None
         try:
             denv = self.denv  # XXX
-            ctval = arg.default.compile_time_value(self.denv)
-            return '%r' % ctval
+            ctval = default_val.compile_time_value(self.denv)
+            repr_val = '%r' % ctval
+            if isinstance(default_val, ExprNodes.UnicodeNode):
+                if repr_val[:1] != 'u':
+                    return u'u%s' % repr_val
+            elif isinstance(default_val, ExprNodes.BytesNode):
+                if repr_val[:1] != 'b':
+                    return u'b%s' % repr_val
+            elif isinstance(default_val, ExprNodes.StringNode):
+                if repr_val[:1] in ('u', 'b'):
+                    repr_val[1:]
+            return repr_val
         except Exception:
             try:
-                return arg.default.name # XXX
+                return default_val.name # XXX
             except AttributeError:
                 return '<???>'
 
@@ -82,7 +91,7 @@ class EmbedSignature(CythonTransform):
 
     def _embed_signature(self, signature, node_doc):
         if node_doc:
-            return signature + '\n' + node_doc
+            return "%s\n%s" % (signature, node_doc)
         else:
             return signature
 

Cython/Compiler/Buffer.py

-from Cython.Compiler.Visitor import VisitorTransform, CythonTransform
-from Cython.Compiler.ModuleNode import ModuleNode
-from Cython.Compiler.Nodes import *
-from Cython.Compiler.ExprNodes import *
-from Cython.Compiler.StringEncoding import EncodedString
-from Cython.Compiler.Errors import CompileError
-from Cython.Utils import UtilityCode
+from Visitor import VisitorTransform, CythonTransform
+from ModuleNode import ModuleNode
+from Nodes import *
+from ExprNodes import *
+from StringEncoding import EncodedString
+from Errors import CompileError
+from Code import UtilityCode
 import Interpreter
 import PyrexTypes
 
@@ -146,7 +146,7 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee
     for name, (value, pos) in dictargs.iteritems():
         if not name in buffer_options:
             raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name)
-        options[name.encode("ASCII")] = value
+        options[name] = value
 
     for name, (value, pos) in zip(buffer_options, posargs):
         if not name in buffer_options:
@@ -245,8 +245,9 @@ def put_acquire_arg_buffer(entry, code, pos):
     # need to care about the buffer then.
     put_unpack_buffer_aux_into_scope(buffer_aux, entry.type.mode, code)
 
-def get_release_buffer_code(entry):
-    return "__Pyx_SafeReleaseBuffer(&%s)" % entry.buffer_aux.buffer_info_var.cname
+def put_release_buffer_code(code, entry):
+    code.globalstate.use_utility_code(acquire_utility_code)
+    code.putln("__Pyx_SafeReleaseBuffer(&%s);" % entry.buffer_aux.buffer_info_var.cname)
 
 def get_getbuffer_call(code, obj_cname, buffer_aux, buffer_type):
     ndim = buffer_type.ndim
@@ -432,7 +433,7 @@ def use_empty_bufstruct_code(env, max_ndim):
         Py_ssize_t __Pyx_zeros[] = {%s};
         Py_ssize_t __Pyx_minusones[] = {%s};
     """) % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim))
-    env.use_utility_code(UtilityCode(proto=code), "empty_bufstruct_code")
+    env.use_utility_code(UtilityCode(proto=code))
 
 
 def buf_lookup_full_code(proto, defin, name, nd):
@@ -494,7 +495,6 @@ def use_py2_buffer_functions(env):
     # Emulation of PyObject_GetBuffer and PyBuffer_Release for Python 2.
     # For >= 2.6 we do double mode -- use the new buffer interface on objects
     # which has the right tp_flags set, but emulation otherwise.
-    codename = "PyObject_GetBuffer" # just a representative unique key
 
     # Search all types for __getbuffer__ overloads
     types = []
@@ -567,7 +567,7 @@ def use_py2_buffer_functions(env):
         #define __Pyx_GetBuffer PyObject_GetBuffer
         #define __Pyx_ReleaseBuffer PyBuffer_Release
         #endif
-    """), impl = code), codename)
+    """), impl = code))
 
 
 def mangle_dtype_name(dtype):

Cython/Compiler/Builtin.py

@@ -3,7 +3,7 @@
 #
 
 from Symtab import BuiltinScope, StructOrUnionScope
-from Cython.Utils import UtilityCode
+from Code import UtilityCode
 from TypeSlots import Signature
 import PyrexTypes
 import Naming
@@ -21,14 +21,14 @@ builtin_function_table = [
     #('eval',      "",     "",      ""),
     #('execfile',  "",     "",      ""),
     #('filter',    "",     "",      ""),
-    ('getattr',    "OO",   "O",     "PyObject_GetAttr"),
+    #('getattr',    "OO",   "O",     "PyObject_GetAttr"),   # optimised later on
     ('getattr3',   "OOO",  "O",     "__Pyx_GetAttr3",       "getattr"),
     ('hasattr',    "OO",   "b",     "PyObject_HasAttr"),
     ('hash',       "O",    "l",     "PyObject_Hash"),
     #('hex',       "",     "",      ""),
     #('id',        "",     "",      ""),
     #('input',     "",     "",      ""),
-    ('intern',     "s",    "O",     "__Pyx_InternFromString"),
+    ('intern',     "O",    "O",     "__Pyx_Intern"),
     ('isinstance', "OO",   "b",     "PyObject_IsInstance"),
     ('issubclass', "OO",   "b",     "PyObject_IsSubclass"),
     ('iter',       "O",    "O",     "PyObject_GetIter"),
@@ -50,7 +50,7 @@ builtin_function_table = [
     #('round',     "",     "",      ""),
     ('setattr',    "OOO",  "r",     "PyObject_SetAttr"),
     #('sum',       "",     "",      ""),
-    ('type',       "O",    "O",     "PyObject_Type"),
+    #('type',       "O",    "O",     "PyObject_Type"),
     #('unichr',    "",     "",      ""),
     #('unicode',   "",     "",      ""),
     #('vars',      "",     "",      ""),
@@ -103,10 +103,7 @@ builtin_types_table = [
 
     ("tuple",   "PyTuple_Type",    []),
 
-    ("list",    "PyList_Type",     [("append", "OO",   "i", "PyList_Append"),
-                                    ("insert", "OZO",  "i", "PyList_Insert"),
-#                                    ("sort",   "O",    "i", "PyList_Sort"), # has optional arguments
-                                    ("reverse","O",    "i", "PyList_Reverse")]),
+    ("list",    "PyList_Type",     [("insert", "OZO",  "i", "PyList_Insert")]),
 
     ("dict",    "PyDict_Type",     [("items", "O",   "O", "PyDict_Items"),
                                     ("keys",  "O",   "O", "PyDict_Keys"),
@@ -123,6 +120,13 @@ builtin_types_table = [
     ("frozenset", "PyFrozenSet_Type", []),
 ]
 
+types_that_construct_their_instance = (
+    # some builtin types do not always return an instance of
+    # themselves - these do:
+    'type', 'bool', 'long', 'float', 'bytes', 'unicode', 'tuple', 'list',
+    'dict', 'file', 'set', 'frozenset'
+    # 'str',             # only in Py3.x
+    )
 
         
 builtin_structs_table = [
@@ -163,6 +167,14 @@ bad:
 
 pyexec_utility_code = UtilityCode(
 proto = """
+#if PY_VERSION_HEX < 0x02040000
+#ifndef Py_COMPILE_H
+#include "compile.h"
+#endif
+#ifndef Py_EVAL_H
+#include "eval.h"
+#endif
+#endif
 static PyObject* __Pyx_PyRun(PyObject*, PyObject*, PyObject*);
 """,
 impl = """
@@ -171,18 +183,36 @@ static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
     PyObject* s = 0;
     char *code = 0;
 
-    if (!locals && !globals) {
+    if (!globals || globals == Py_None) {
         globals = PyModule_GetDict(%s);""" % Naming.module_cname + """
         if (!globals)
             goto bad;
+    } else if (!PyDict_Check(globals)) {
+        PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.100s",
+                     globals->ob_type->tp_name);
+        goto bad;
+    }
+    if (!locals || locals == Py_None) {
         locals = globals;
-    } else if (!locals) {
-        locals = globals;
-    } else if (!globals) {
-        globals = locals;
     }
 
+
+    if (PyDict_GetItemString(globals, "__builtins__") == NULL) {
+	PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins());
+    }
+
+    if (PyCode_Check(o)) {
+        if (PyCode_GetNumFree((PyCodeObject *)o) > 0) {
+            PyErr_SetString(PyExc_TypeError,
+                "code object passed to exec() may not contain free variables");
+            goto bad;
+        }
+	result = PyEval_EvalCode((PyCodeObject *)o, globals, locals);
+    } else {
+        PyCompilerFlags cf;
+        cf.cf_flags = 0;
 	if (PyUnicode_Check(o)) {
+            cf.cf_flags = PyCF_SOURCE_IS_UTF8;
     	    s = PyUnicode_AsUTF8String(o);
     	    if (!s) goto bad;
     	    o = s;
@@ -191,20 +221,23 @@ static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
 	#else
 	} else if (!PyString_Check(o)) {
 	#endif
-        /* FIXME: support file objects and code objects */
     	    PyErr_SetString(PyExc_TypeError,
-            "exec currently requires a string as code input.");
+        	"exec: arg 1 must be string, bytes or code object");
     	    goto bad;
 	}
-
 	#if PY_MAJOR_VERSION >= 3
 	code = PyBytes_AS_STRING(o);
 	#else
 	code = PyString_AS_STRING(o);
 	#endif
+	if (PyEval_MergeCompilerFlags(&cf)) {
+	    result = PyRun_StringFlags(code, Py_file_input, globals, locals, &cf);
+        } else {
 	    result = PyRun_String(code, Py_file_input, globals, locals);
-
+        }
         Py_XDECREF(s);
+    }
+
     return result;
 bad:
     Py_XDECREF(s);
@@ -214,12 +247,23 @@ bad:
 
 intern_utility_code = UtilityCode(
 proto = """
-#if PY_MAJOR_VERSION >= 3
-#  define __Pyx_InternFromString(s) PyUnicode_InternFromString(s)
-#else
-#  define __Pyx_InternFromString(s) PyString_InternFromString(s)
-#endif
-""")
+static PyObject* __Pyx_Intern(PyObject* s); /* proto */
+""",
+impl = '''
+static PyObject* __Pyx_Intern(PyObject* s) {
+    if (!(likely(PyString_CheckExact(s)))) {
+        PyErr_Format(PyExc_TypeError, "Expected str, got %s", Py_TYPE(s)->tp_name);
+        return 0;
+    }
+    Py_INCREF(s);
+    #if PY_MAJOR_VERSION >= 3
+    PyUnicode_InternInPlace(&s);
+    #else
+    PyString_InternInPlace(&s);
+    #endif
+    return s;
+}
+''')
 
 def put_py23_set_init_utility_code(code, pos):
     code.putln("#if PY_VERSION_HEX < 0x02040000")
@@ -369,13 +413,16 @@ def init_builtins():
     init_builtin_funcs()
     init_builtin_types()
     init_builtin_structs()
-    global list_type, tuple_type, dict_type, set_type, bytes_type, unicode_type, type_type
+    global list_type, tuple_type, dict_type, set_type, type_type
+    global bytes_type, str_type, unicode_type, float_type
     type_type  = builtin_scope.lookup('type').type
     list_type  = builtin_scope.lookup('list').type
     tuple_type = builtin_scope.lookup('tuple').type
     dict_type  = builtin_scope.lookup('dict').type
     set_type   = builtin_scope.lookup('set').type
     bytes_type = builtin_scope.lookup('bytes').type
+    str_type   = builtin_scope.lookup('str').type
     unicode_type = builtin_scope.lookup('unicode').type
+    float_type = builtin_scope.lookup('float').type
 
 init_builtins()

Cython/Compiler/CmdLine.py

@@ -17,7 +17,6 @@ Options:
   -I, --include-dir <directory>  Search for include files in named directory
                                  (multiply include directories are allowed).
   -o, --output-file <filename>   Specify name of generated C file
-  -r, --recursive                Recursively find and compile dependencies
   -t, --timestamps               Only compile newer source files (implied with -r)
   -f, --force                    Compile all source files (overrides implied -t)
   -q, --quiet                    Don't print module names in recursive mode
@@ -27,8 +26,6 @@ Options:
   -z, --pre-import <module>      If specified, assume undeclared names in this 
                                  module. Emulates the behavior of putting 
                                  "from <module> import *" at the top of the file. 
-  --incref-local-binop           Force local an extra incref on local variables before
-                                 performing any binary operations.
   --cleanup <level>              Release interned objects on python exit, for memory debugging. 
                                  Level indicates aggressiveness, default 0 releases nothing. 
   -w, --working <directory>      Sets the working directory for Cython (the directory modules 
@@ -39,12 +36,16 @@ Options:
   --line-directives              Produce #line directives pointing to the .pyx source
   --cplus                        Output a c++ rather than c file.
   --embed                        Embed the Python interpreter in a main() method.
-  --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
+  -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
 """
 
+# The following is broken http://trac.cython.org/cython_trac/ticket/379
+#  -r, --recursive                Recursively find and compile dependencies
+
+
 #The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
-#  -X, --link       Link .o file to produce extension module (implies -C)
+#  --link           Link .o file to produce extension module (implies -C)
 #  -+, --cplus      Use C++ compiler for compiling and linking
 #  Additional .o files to link may be supplied when using -X."""
 
@@ -81,11 +82,7 @@ def parse_command_line(args):
                 options.use_listing_file = 1
             elif option in ("-C", "--compile"):
                 options.c_only = 0
-            elif option in ("-X", "--link"):
-                if option == "-X":
-                    print >>sys.stderr, "Deprecation warning: The -X command line switch will be changed to a"
-                    print >>sys.stderr, "shorthand for --directive in Cython 0.12. Please use --link instead."
-                    print >>sys.stderr
+            elif option in ("--link"):
                 options.c_only = 0
                 options.obj_only = 0
             elif option in ("-+", "--cplus"):
@@ -112,8 +109,6 @@ def parse_command_line(args):
                 Options.embed_pos_in_docstring = 1
             elif option in ("-z", "--pre-import"):
                 Options.pre_import = pop_arg()
-            elif option == "--incref-local-binop":
-                Options.incref_local_binop = 1
             elif option == "--cleanup":
                 Options.generate_cleanup_code = int(pop_arg())
             elif option in ("-D", "--no-docstrings"):
@@ -126,11 +121,12 @@ def parse_command_line(args):
                 options.emit_linenums = True
             elif option in ("-X", "--directive"):
                 try:
-                    options.pragma_overrides = Options.parse_option_list(pop_arg())
+                    options.compiler_directives = Options.parse_directive_list(pop_arg(), relaxed_bool=True)
                 except ValueError, e:
                     sys.stderr.write("Error in compiler directive: %s\n" % e.message)
                     sys.exit(1)
             else:
+                sys.stderr.write("Unknown compiler flag: %s\n" % option)
                 bad_usage()
         else:
             arg = pop_arg()

Cython/Compiler/ControlFlow.py

@@ -13,7 +13,13 @@ import bisect, sys
 # redesigned. It doesn't take return, raise, continue, or break into 
 # account. 
 
-_END_POS = ((unichr(sys.maxunicode)*10),())
+from Cython.Compiler.Scanning import StringSourceDescriptor
+try:
+    _END_POS = (StringSourceDescriptor(unichr(sys.maxunicode)*10, ''),
+                sys.maxint, sys.maxint)
+except AttributeError: # Py3
+    _END_POS = (StringSourceDescriptor(unichr(sys.maxunicode)*10, ''),
+                sys.maxsize, sys.maxsize)
 
 class ControlFlow(object):
 
@@ -51,10 +57,36 @@ class ControlFlow(object):
             try:
                 return self.tip[item]
             except KeyError:
-                self.tip[item] = pos_state = self._get_pos_state(item, pos)
+                pass
+        pos_state = self._get_pos_state(item, pos)
+        if pos > self.end_pos:
+            self.tip[item] = pos_state
         return pos_state
-        else:
-            return self._get_pos_state(item, pos)
+
+    def _get_pos_state(self, item, pos):
+        current = self
+        while current is not None and pos <= current.start_pos:
+            current = current.incoming
+        if current is None:
+            return (None, None)
+        state = current._get_pos_state_local(item, pos)
+        while state is None and current.incoming is not None:
+            current = current.incoming
+            state = current._get_pos_state_local(item, pos)
+        if state is None:
+            return (None, None)
+        return state
+
+    def set_state(self, pos, item, state):
+        if item in self.tip:
+            del self.tip[item]
+        current = self
+        while pos < current.start_pos and current.incoming is not None:
+            current = current.incoming
+            if item in current.tip:
+                del current.tip[item]
+        current._set_state_local(pos, item, state)
+        
         
 class LinearControlFlow(ControlFlow):
 
@@ -62,13 +94,7 @@ class LinearControlFlow(ControlFlow):
         ControlFlow.__init__(self, start_pos, incoming, parent)
         self.events = {}
             
-    def set_state(self, pos, item, state):
-        if item in self.tip:
-            del self.tip[item]
-        if pos < self.start_pos:
-            if self.incoming is not None:
-                self.incoming.set_state(pos, item, state)
-        else:
+    def _set_state_local(self, pos, item, state):
         if item in self.events:
             event_list = self.events[item]
         else:
@@ -76,20 +102,13 @@ class LinearControlFlow(ControlFlow):
             self.events[item] = event_list
         bisect.insort(event_list, (pos, state))
 
-        
-    def _get_pos_state(self, item, pos):
-        if pos > self.start_pos:
+    def _get_pos_state_local(self, item, pos):
         if item in self.events:
             event_list = self.events[item]
             for event in event_list[::-1]:
                 if event[0] < pos:
                     return event
-                        
-        if self.incoming is not None:
-            return self.incoming.get_pos_state(item, pos)
-        
-        else:
-            return None, None
+        return None
 
     def to_string(self, indent='', limit=None):
     
@@ -116,37 +135,35 @@ class BranchingControlFlow(ControlFlow):
         self.branches = [LinearControlFlow(start_pos, incoming, parent=self)]
         self.branch_starts = [start_pos]
         
-    def set_state(self, pos, item, state):
-    
-        if item in self.tip:
-            del self.tip[item]
-        
-        if pos < self.start_pos:
-            self.incoming.set_state(pos, item, state)
-        else:
+    def _set_state_local(self, pos, item, state):
         for branch_pos, branch in zip(self.branch_starts[::-1], self.branches[::-1]):
             if pos >= branch_pos:
-                    branch.set_state(pos, item, state)
+                branch._set_state_local(pos, item, state)
                 return
     
-    def _get_pos_state(self, item, pos):
-        if pos <= self.start_pos:
-            return self.incoming.get_pos_state(item, pos)
-        elif pos < self.end_pos:
+    def _get_pos_state_local(self, item, pos, stop_at=None):
+        if pos < self.end_pos:
             for branch_pos, branch in zip(self.branch_starts[::-1], self.branches[::-1]):
                 if pos >= branch_pos:
-                    return branch.get_pos_state(item, pos)
+                    return branch._get_pos_state_local(item, pos)
         else:
-            last_pos, last_state = self.branches[0].get_pos_state(item, pos)
+            state = self.branches[0]._get_pos_state_local(item, pos)
+            if state is None:
+                return None, None
+            last_pos, last_state = state
             if last_state is None:
                 return None, None
             for branch in self.branches[1:]:
-                other_pos, other_state = branch.get_pos_state(item, pos)
-                if other_state is None or other_state != last_state:
+                state = branch._get_pos_state_local(item, pos)
+                if state is None:
+                    return None, None
+                other_pos, other_state = state
+                if other_state != last_state:
                     return None, None
                 elif last_pos is not other_pos:
                     last_pos = max(last_pos, other_pos)
             return last_pos, last_state
+        return None
 
     def new_branch(self, pos):
         self.branches.append(LinearControlFlow(pos, self.incoming, parent=self))

Cython/Compiler/DebugFlags.py

@@ -8,3 +8,9 @@ debug_temp_code_comments = 0
 
 # Write a call trace of the code generation phase into the C code
 debug_trace_code_generation = 0
+
+# Do not replace exceptions with user-friendly error messages
+debug_no_exception_intercept = 1
+
+# Print a message each time a new stage in the pipeline is entered
+debug_verbose_pipeline = 0

Cython/Compiler/Errors.py

@@ -21,28 +21,29 @@ def context(position):
         F = list(source.get_lines())
     except UnicodeDecodeError:
         # file has an encoding problem
-        s = "[unprintable code]\n"
+        s = u"[unprintable code]\n"
     else:
-        s =''.join(F[max(0, position[1]-6):position[1]])
-        s = '...\n' + s + ' '*(position[2]-1) + '^\n'
-    s = '-'*60 + '\n' + s + '-'*60 + '\n'
+        s = u''.join(F[max(0, position[1]-6):position[1]])
+        s = u'...\n%s%s^\n' % (s, u' '*(position[2]-1))
+    s = u'%s\n%s%s\n' % (u'-'*60, s, u'-'*60)
     return s
 
 class CompileError(PyrexError):
     
-    def __init__(self, position = None, message = ""):
+    def __init__(self, position = None, message = u""):
         self.position = position
         self.message_only = message
         self.reported = False
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
-            pos_str = "%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
+            pos_str = u"%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
             cont = context(position)
         else:
-            pos_str = ""
-            cont = ''
-        Exception.__init__(self, '\nError converting Pyrex file to C:\n' + cont + '\n' + pos_str + message )
+            pos_str = u""
+            cont = u''
+        Exception.__init__(self, u'\nError converting Pyrex file to C:\n%s\n%s%s' % (
+            cont, pos_str, message))
 
 class CompileWarning(PyrexWarning):
     
@@ -51,9 +52,9 @@ class CompileWarning(PyrexWarning):
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
         if position:
-            pos_str = "%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
+            pos_str = u"%s:%d:%d: " % (position[0].get_description(), position[1], position[2])
         else:
-            pos_str = ""
+            pos_str = u""
         Exception.__init__(self, pos_str + message)
 
 
@@ -61,7 +62,7 @@ class InternalError(Exception):
     # If this is ever raised, there is a bug in the compiler.
     
     def __init__(self, message):
-        Exception.__init__(self, "Internal compiler error: %s"
+        Exception.__init__(self, u"Internal compiler error: %s"
             % message)
 
 
@@ -73,7 +74,7 @@ class CompilerCrash(CompileError):
         else:
             message = u'\n'
         if context:
-            message = "Compiler crash in " + context + message
+            message = u"Compiler crash in %s%s" % (context, message)
         if stacktrace:
             import traceback
             message += (
@@ -118,11 +119,15 @@ def report_error(err):
         # See Main.py for why dual reporting occurs. Quick fix for now.
         if err.reported: return
         err.reported = True
-        line = "%s\n" % err
+        line = u"%s\n" % err
         if listing_file:
-            listing_file.write(line)
+            try: listing_file.write(line)
+            except UnicodeEncodeError:
+                listing_file.write(line.encode('ASCII', 'replace'))
         if echo_file:
-            echo_file.write(line)
+            try: echo_file.write(line)
+            except UnicodeEncodeError:
+                echo_file.write(line.encode('ASCII', 'replace'))
         num_errors = num_errors + 1
 
 def error(position, message):
@@ -145,6 +150,20 @@ def warning(position, message, level=0):
         echo_file.write(line)
     return warn
 
+_warn_once_seen = {}
+def warn_once(position, message, level=0):
+    if level < LEVEL or message in _warn_once_seen:
+        return
+    warn = CompileWarning(position, message)
+    line = "warning: %s\n" % warn
+    if listing_file:
+        listing_file.write(line)
+    if echo_file:
+        echo_file.write(line)
+    _warn_once_seen[message] = True
+    return warn
+
+
 # These functions can be used to momentarily suppress errors. 
 
 error_stack = []

Cython/Compiler/Interpreter.py

@@ -8,7 +8,6 @@ compile-time values.
 
 from Nodes import *
 from ExprNodes import *
-from Visitor import BasicVisitor
 from Errors import CompileError
 
 

Cython/Compiler/Main.py

@@ -24,6 +24,7 @@ from Symtab import BuiltinScope, ModuleScope
 from Cython import Utils
 from Cython.Utils import open_new_file, replace_suffix
 import CythonScope
+import DebugFlags
 
 module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
 
@@ -34,6 +35,12 @@ def dumptree(t):
     print t.dump()
     return t
 
+def abort_on_errors(node):
+    # Stop the pipeline if there are any errors.
+    if Errors.num_errors != 0:
+        raise InternalError, "abort"
+    return node
+
 class CompilationData(object):
     #  Bundles the information that is passed from transform to transform.
     #  (For now, this is only)
@@ -59,16 +66,17 @@ class Context(object):
     #  include_directories   [string]
     #  future_directives     [object]
     
-    def __init__(self, include_directories, pragma_overrides, cpp=False):
+    def __init__(self, include_directories, compiler_directives, cpp=False):
         #self.modules = {"__builtin__" : BuiltinScope()}
         import Builtin, CythonScope
         self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.modules["cython"] = CythonScope.create_cython_scope(self)
         self.include_directories = include_directories
         self.future_directives = set()
-        self.pragma_overrides = pragma_overrides
+        self.compiler_directives = compiler_directives
         self.cpp = cpp
 
+
         self.pxds = {} # full name -> node tree
 
         standard_include_path = os.path.abspath(
@@ -81,12 +89,16 @@ class Context(object):
         from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
         from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
         from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
-        from ParseTreeTransforms import AlignFunctionDefinitions
+        from TypeInference import MarkAssignments
+        from ParseTreeTransforms import AlignFunctionDefinitions, GilCheck
+        from AnalysedTreeTransforms import AutoTestDictTransform
         from AutoDocTransforms import EmbedSignature
         from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
-        from Optimize import FlattenBuiltinTypeCreation, ConstantFolding, FinalOptimizePhase
+        from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
+        from Optimize import ConstantFolding, FinalOptimizePhase
+        from Optimize import DropRefcountingTransform
         from Buffer import IntroduceBufferAuxiliaryVars
-        from ModuleNode import check_c_declarations
+        from ModuleNode import check_c_declarations, check_c_declarations_pxd
 
         # Temporary hack that can be used to ensure that all result_code's
         # are generated at code generation time.
@@ -98,7 +110,7 @@ class Context(object):
                 return node
 
         if pxd:
-            _check_c_declarations = None
+            _check_c_declarations = check_c_declarations_pxd
             _specific_post_parse = PxdPostParse(self)
         else:
             _check_c_declarations = check_c_declarations
@@ -113,22 +125,27 @@ class Context(object):
             NormalizeTree(self),
             PostParse(self),
             _specific_post_parse,
-            InterpretCompilerDirectives(self, self.pragma_overrides),
+            InterpretCompilerDirectives(self, self.compiler_directives),
             _align_function_definitions,
+            ConstantFolding(),
             FlattenInListTransform(),
             WithTransform(self),
             DecoratorTransform(self),
             AnalyseDeclarationsTransform(self),
+            AutoTestDictTransform(self),
             EmbedSignature(self),
+            EarlyReplaceBuiltinCalls(self),
+            MarkAssignments(self),
             TransformBuiltinMethods(self),
             IntroduceBufferAuxiliaryVars(self),
             _check_c_declarations,
             AnalyseExpressionsTransform(self),
-            FlattenBuiltinTypeCreation(),
-            ConstantFolding(),
+            OptimizeBuiltinCalls(self),
             IterationTransform(),
             SwitchTransform(),
+            DropRefcountingTransform(),
             FinalOptimizePhase(self),
+            GilCheck(),
 #            ClearResultCodes(self),
 #            SpecialFunctions(self),
             #        CreateClosureClasses(context),
@@ -154,10 +171,16 @@ class Context(object):
                  module_node.scope.utility_code_list.extend(scope.utility_code_list)
             return module_node
 
+        test_support = []
+        if options.evaluate_tree_assertions:
+            from Cython.TestUtils import TreeAssertVisitor
+            test_support.append(TreeAssertVisitor())
+
         return ([
                 create_parse(self),
-            ] + self.create_pipeline(pxd=False, py=py) + [
+            ] + self.create_pipeline(pxd=False, py=py) + test_support + [
                 inject_pxd_code,
+                abort_on_errors,
                 generate_pyx_code,
             ])
 
@@ -190,20 +213,24 @@ class Context(object):
         return Errors.report_error(exc)
 
     def run_pipeline(self, pipeline, source):
-        err = None
+        error = None
         data = source
         try:
             for phase in pipeline:
                 if phase is not None:
+                    if DebugFlags.debug_verbose_pipeline:
+                        print "Entering pipeline phase %r" % phase
                     data = phase(data)
         except CompileError, err:
             # err is set
             Errors.report_error(err)
+            error = err
         except InternalError, err:
             # Only raise if there was not an earlier error
             if Errors.num_errors == 0:
                 raise
-        return (err, data)
+            error = err
+        return (error, data)
 
     def find_module(self, module_name, 
             relative_to = None, pos = None, need_pxd = 1):
@@ -314,7 +341,9 @@ class Context(object):
             else:
                 dirs = [self.find_root_package_dir(file_desc.filename)] + dirs
 
-        dotted_filename = qualified_name + suffix
+        dotted_filename = qualified_name
+        if suffix:
+            dotted_filename += suffix
         if not include:
             names = qualified_name.split('.')
             package_names = names[:-1]
@@ -523,7 +552,7 @@ def create_default_resultobj(compilation_source, options):
 
 def run_pipeline(source, options, full_module_name = None):
     # Set up context
-    context = Context(options.include_path, options.pragma_overrides, options.cplus)
+    context = Context(options.include_path, options.compiler_directives, options.cplus)
 
     # Set up source object
     cwd = os.getcwd()
@@ -576,7 +605,8 @@ class CompilationOptions(object):
                                 defaults to true when recursive is true.
     verbose           boolean   Always print source names being compiled
     quiet             boolean   Don't print source names in recursive mode
-    pragma_overrides  dict      Overrides for pragma options (see Options.py)
+    compiler_directives  dict      Overrides for pragma options (see Options.py)
+    evaluate_tree_assertions boolean  Test support: evaluate parse tree assertions
     
     Following options are experimental and only used on MacOSX:
     
@@ -716,6 +746,8 @@ def compile(source, options = None, c_compile = 0, c_link = 0,
 #  Main command-line entry point
 #
 #------------------------------------------------------------------------
+def setuptools_main():
+    return main(command_line = 1)
 
 def main(command_line = 0):
     args = sys.argv[1:]
@@ -764,7 +796,8 @@ default_options = dict(
     timestamps = None,
     verbose = 0,
     quiet = 0,
-    pragma_overrides = {},
+    compiler_directives = {},
+    evaluate_tree_assertions = False,
     emit_linenums = False,
 )
 if sys.platform == "mac":

Cython/Compiler/Naming.py

@@ -27,6 +27,7 @@ label_prefix      = pyrex_prefix + "L"
 pymethdef_prefix  = pyrex_prefix + "mdef_"
 methtab_prefix    = pyrex_prefix + "methods_"
 memtab_prefix     = pyrex_prefix + "members_"
+interned_str_prefix = pyrex_prefix + "n_"
 interned_num_prefix = pyrex_prefix + "int_"
 objstruct_prefix  = pyrex_prefix + "obj_"
 typeptr_prefix    = pyrex_prefix + "ptype_"
@@ -74,6 +75,7 @@ c_api_tab_cname  = pyrex_prefix + "c_api_tab"
 gilstate_cname   = pyrex_prefix + "state"
 skip_dispatch_cname = pyrex_prefix + "skip_dispatch"
 empty_tuple      = pyrex_prefix + "empty_tuple"
+empty_bytes      = pyrex_prefix + "empty_bytes"
 print_function   = pyrex_prefix + "print"
 print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
@@ -83,6 +85,8 @@ import_star      = pyrex_prefix + "import_star"
 import_star_set  = pyrex_prefix + "import_star_set"
 cur_scope_cname  = pyrex_prefix + "cur_scope"
 enc_scope_cname  = pyrex_prefix + "enc_scope"
+frame_cname      = pyrex_prefix + "frame"
+frame_code_cname = pyrex_prefix + "frame_code"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Options.py

@@ -11,12 +11,6 @@ cplus = 0
 pre_import = None
 docstrings = True
 
-# This is a SAGE-specific option that will 
-# cause Cython to incref local variables before
-# performing a binary operation on them, for 
-# safe detection of inplace operators. 
-incref_local_binop = 0
-
 # Decref global variables in this module on exit for garbage collection. 
 # 0: None, 1+: interned objects, 2+: cdef globals, 3+: types objects
 # Mostly for reducing noise for Valgrind, only executes at process exit
@@ -56,75 +50,98 @@ embed = False
 
 
 # Declare compiler directives
-option_defaults = {
+directive_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
     'embedsignature' : False,
     'locals' : {},
     'auto_cpdef': False,
-    'cdivision': True,  # Will be False in 0.12
+    'cdivision': False, # was True before 0.12
     'cdivision_warnings': False,
     'always_allow_keywords': False,
     'wraparound' : True,
-    'c99_complex' : False, # Don't use macro wrappers for complex arith, not sure what to name this...
+    'ccomplex' : False, # use C99/C++ for complex types and arith
     'callspec' : "",
+    'profile': False,
+    'infer_types': False,
+    'autotestdict': True,
+
+# test support
+    'test_assert_path_exists' : [],
+    'test_fail_if_path_exists' : [],
 }
 
 # Override types possibilities above, if needed
-option_types = { }
-
-for key, val in option_defaults.items():
-    if key not in option_types:
-        option_types[key] = type(val)
+directive_types = {
+    'infer_types' : bool, # values can be True/None/False
+    }
+
+for key, val in directive_defaults.items():
+    if key not in directive_types:
+        directive_types[key] = type(val)
+
+directive_scopes = { # defaults to available everywhere
+    # 'module', 'function', 'class', 'with statement'
+    'autotestdict' : ('module',),
+    'test_assert_path_exists' : ('function',),
+    'test_fail_if_path_exists' : ('function',),
+}
 
-def parse_option_value(name, value):
+def parse_directive_value(name, value, relaxed_bool=False):
     """
     Parses value as an option value for the given name and returns
     the interpreted value. None is returned if the option does not exist.
 
-    >>> print parse_option_value('nonexisting', 'asdf asdfd')
+    >>> print parse_directive_value('nonexisting', 'asdf asdfd')
     None
-    >>> parse_option_value('boundscheck', 'True')
+    >>> parse_directive_value('boundscheck', 'True')
     True
-    >>> parse_option_value('boundscheck', 'true')
+    >>> parse_directive_value('boundscheck', 'true')
     Traceback (most recent call last):
        ...
     ValueError: boundscheck directive must be set to True or False
     
     """
-    type = option_types.get(name)
+    type = directive_types.get(name)
     if not type: return None
     if type is bool:
-        if value == "True": return True
-        elif value == "False": return False
-        else: raise ValueError("%s directive must be set to True or False" % name)
+        value = str(value)
+        if value == 'True': return True
+        if value == 'False': return False
+        if relaxed_bool:
+            value = value.lower()
+            if value in ("true", "yes"): return True
+            elif value in ("false", "no"): return False
+        raise ValueError("%s directive must be set to True or False" % name)
     elif type is int:
         try:
             return int(value)
         except ValueError:
             raise ValueError("%s directive must be set to an integer" % name)
+    elif type is str:
+        return str(value)
     else:
         assert False
 
-def parse_option_list(s):
+def parse_directive_list(s, relaxed_bool=False, ignore_unknown=False):
     """
     Parses a comma-seperated list of pragma options. Whitespace
     is not considered.
 
-    >>> parse_option_list('      ')
+    >>> parse_directive_list('      ')
     {}
-    >>> (parse_option_list('boundscheck=True') ==
+    >>> (parse_directive_list('boundscheck=True') ==
     ... {'boundscheck': True})
     True
-    >>> parse_option_list('  asdf')
+    >>> parse_directive_list('  asdf')
     Traceback (most recent call last):
        ...
     ValueError: Expected "=" in option "asdf"
-    >>> parse_option_list('boundscheck=hey')
+    >>> parse_directive_list('boundscheck=hey')
     Traceback (most recent call last):
        ...
-    ValueError: Must pass a boolean value for option "boundscheck"
-    >>> parse_option_list('unknown=True')
+    ValueError: boundscheck directive must be set to True or False
+    >>> parse_directive_list('unknown=True')
     Traceback (most recent call last):
        ...
     ValueError: Unknown option: "unknown"
@@ -134,19 +151,11 @@ def parse_option_list(s):
         item = item.strip()
         if not item: continue
         if not '=' in item: raise ValueError('Expected "=" in option "%s"' % item)
-        name, value = item.strip().split('=')
-        try:
-            type = option_types[name]
-        except KeyError:
+        name, value = [ s.strip() for s in item.strip().split('=', 1) ]
+        parsed_value = parse_directive_value(name, value, relaxed_bool=relaxed_bool)
+        if parsed_value is None:
+            if not ignore_unknown:
                 raise ValueError('Unknown option: "%s"' % name)
-        if type is bool:
-            value = value.lower()
-            if value in ('true', 'yes'):
-                value = True
-            elif value in ('false', 'no'):
-                value = False
-            else: raise ValueError('Must pass a boolean value for option "%s"' % name)
-            result[name] = value
         else:
-            assert False
+            result[name] = parsed_value
     return result

Cython/Compiler/Parsing.pxd

@@ -16,6 +16,7 @@ cpdef p_not_test(PyrexScanner s)
 cpdef p_comparison(PyrexScanner s)
 cpdef p_cascaded_cmp(PyrexScanner s)
 cpdef p_cmp_op(PyrexScanner s)
+cpdef p_starred_expr(PyrexScanner s)
 cpdef p_bit_expr(PyrexScanner s)
 cpdef p_xor_expr(PyrexScanner s)
 cpdef p_and_expr(PyrexScanner s)
@@ -59,7 +60,6 @@ cpdef p_testlist(PyrexScanner s)
 #-------------------------------------------------------
 
 cpdef flatten_parallel_assignments(input, output)
-cpdef find_parallel_assignment_size(input)
 
 cpdef p_global_statement(PyrexScanner s)
 cpdef p_expression_or_assignment(PyrexScanner s)
@@ -142,7 +142,7 @@ cpdef p_ctypedef_statement(PyrexScanner s, ctx)
 cpdef p_decorators(PyrexScanner s)
 cpdef p_def_statement(PyrexScanner s, decorators = *)
 cpdef p_py_arg_decl(PyrexScanner s)
-cpdef p_class_statement(PyrexScanner s)
+cpdef p_class_statement(PyrexScanner s, decorators)
 cpdef p_c_class_definition(PyrexScanner s, pos,  ctx)
 cpdef p_c_class_options(PyrexScanner s)
 cpdef p_property_decl(PyrexScanner s)

Cython/Compiler/Scanning.py

@@ -9,6 +9,7 @@ import os
 import platform
 import stat
 import sys
+import codecs
 from time import time
 
 import cython
@@ -65,7 +66,7 @@ def hash_source_file(path):
     # tabs by a single space.
     import re
     text = re.sub("[ \t]+", " ", text)
-    hash = new_md5(text).hexdigest()
+    hash = new_md5(text.encode("ASCII")).hexdigest()
     return hash
 
 def open_pickled_lexicon(expected_hash):
@@ -214,14 +215,18 @@ def initial_compile_time_env():
         'UNAME_VERSION', 'UNAME_MACHINE')
     for name, value in zip(names, platform.uname()):
         benv.declare(name, value)
-    import __builtin__
+    import __builtin__ as builtins
     names = ('False', 'True',
         'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate',
         'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min',
         'oct', 'ord', 'pow', 'range', 'reduce', 'repr', 'round', 'slice', 'str',
         'sum', 'tuple', 'xrange', 'zip')
     for name in names:
-        benv.declare(name, getattr(__builtin__, name))
+        try:
+            benv.declare(name, getattr(builtins, name))
+        except AttributeError:
+            # ignore, likely Py3
+            pass
     denv = CompileTimeScope(benv)
     return denv
 
@@ -232,6 +237,7 @@ class SourceDescriptor(object):
     A SourceDescriptor should be considered immutable.
     """
     _escaped_description = None
+    _cmp_name = ''
     def __str__(self):
         assert False # To catch all places where a descriptor is used directly as a filename
     
@@ -241,6 +247,27 @@ class SourceDescriptor(object):
                 self.get_description().encode('ASCII', 'replace').decode("ASCII")
         return self._escaped_description
 
+    def __gt__(self, other):
+        # this is only used to provide some sort of order
+        try:
+            return self._cmp_name > other._cmp_name
+        except AttributeError:
+            return False
+
+    def __lt__(self, other):
+        # this is only used to provide some sort of order
+        try:
+            return self._cmp_name < other._cmp_name
+        except AttributeError:
+            return False
+
+    def __le__(self, other):
+        # this is only used to provide some sort of order
+        try:
+            return self._cmp_name <= other._cmp_name
+        except AttributeError:
+            return False
+
 class FileSourceDescriptor(SourceDescriptor):
     """
     Represents a code source. A code source is a more generic abstraction
@@ -251,9 +278,14 @@ class FileSourceDescriptor(SourceDescriptor):
     """
     def __init__(self, filename):
         self.filename = filename
+        self._cmp_name = filename
     
-    def get_lines(self):
+    def get_lines(self, encoding=None, error_handling=None):
+        if not encoding:
             return Utils.open_source_file(self.filename)
+        else:
+            return codecs.open(self.filename, "rU", encoding=encoding,
+                               errors=error_handling)
     
     def get_description(self):
         return self.filename
@@ -278,9 +310,14 @@ class StringSourceDescriptor(SourceDescriptor):
     def __init__(self, name, code):
         self.name = name
         self.codelines = [x + "\n" for x in code.split("\n")]
+        self._cmp_name = name
     
-    def get_lines(self):
+    def get_lines(self, encoding=None, error_handling=None):
+        if not encoding:
             return self.codelines
+        else:
+            return [ line.encode(encoding, error_handling).decode(encoding)
+                     for line in self.codelines ]
 
     def get_description(self):
         return self.name
@@ -426,7 +463,6 @@ class PyrexScanner(Scanner):
                 sy = systring
             else:
                 systring = EncodedString(systring)
-                systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
         if False: # debug_scanner:

Cython/Compiler/Tests/TestTreePath.py

+import unittest
+from Cython.Compiler.Visitor import PrintTree
+from Cython.TestUtils import TransformTest
+from Cython.Compiler.TreePath import find_first, find_all
+from Cython.Compiler import Nodes, ExprNodes
+
+class TestTreePath(TransformTest):
+    _tree = None
+
+    def _build_tree(self):
+        if self._tree is None:
+            self._tree = self.run_pipeline([], u"""
+            def decorator(fun):  # DefNode
+                return fun       # ReturnStatNode, NameNode
+            @decorator           # NameNode
+            def decorated():     # DefNode
+                pass
+            """)
+        return self._tree
+
+    def test_node_path(self):
+        t = self._build_tree()
+        self.assertEquals(2, len(find_all(t, "//DefNode")))
+        self.assertEquals(2, len(find_all(t, "//NameNode")))
+        self.assertEquals(1, len(find_all(t, "//ReturnStatNode")))
+        self.assertEquals(1, len(find_all(t, "//DefNode//ReturnStatNode")))
+
+    def test_node_path_star(self):
+        t = self._build_tree()
+        self.assertEquals(10, len(find_all(t, "//*")))
+        self.assertEquals(8, len(find_all(t, "//DefNode//*")))
+        self.assertEquals(0, len(find_all(t, "//NameNode//*")))
+
+    def test_node_path_attribute(self):
+        t = self._build_tree()
+        self.assertEquals(2, len(find_all(t, "//NameNode/@name")))
+        self.assertEquals(['fun', 'decorator'], find_all(t, "//NameNode/@name"))
+
+    def test_node_path_attribute_dotted(self):
+        t = self._build_tree()
+        self.assertEquals(1, len(find_all(t, "//ReturnStatNode/@value.name")))
+        self.assertEquals(['fun'], find_all(t, "//ReturnStatNode/@value.name"))
+
+    def test_node_path_child(self):
+        t = self._build_tree()
+        self.assertEquals(1, len(find_all(t, "//DefNode/ReturnStatNode/NameNode")))
+        self.assertEquals(1, len(find_all(t, "//ReturnStatNode/NameNode")))
+
+    def test_node_path_node_predicate(self):
+        t = self._build_tree()
+        self.assertEquals(0, len(find_all(t, "//DefNode[.//ForInStatNode]")))
+        self.assertEquals(2, len(find_all(t, "//DefNode[.//NameNode]")))
+        self.assertEquals(1, len(find_all(t, "//ReturnStatNode[./NameNode]")))
+        self.assertEquals(Nodes.ReturnStatNode,
+                          type(find_first(t, "//ReturnStatNode[./NameNode]")))
+
+    def test_node_path_node_predicate_step(self):
+        t = self._build_tree()
+        self.assertEquals(2, len(find_all(t, "//DefNode[.//NameNode]")))
+        self.assertEquals(8, len(find_all(t, "//DefNode[.//NameNode]//*")))
+        self.assertEquals(1, len(find_all(t, "//DefNode[.//NameNode]//ReturnStatNode")))
+        self.assertEquals(Nodes.ReturnStatNode,
+                          type(find_first(t, "//DefNode[.//NameNode]//ReturnStatNode")))
+
+    def test_node_path_attribute_exists(self):
+        t = self._build_tree()
+        self.assertEquals(2, len(find_all(t, "//NameNode[@name]")))
+        self.assertEquals(ExprNodes.NameNode,
+                          type(find_first(t, "//NameNode[@name]")))
+
+    def test_node_path_attribute_exists_not(self):
+        t = self._build_tree()
+        self.assertEquals(0, len(find_all(t, "//NameNode[not(@name)]")))
+        self.assertEquals(2, len(find_all(t, "//NameNode[not(@honking)]")))
+
+    def test_node_path_and(self):
+        t = self._build_tree()
+        self.assertEquals(1, len(find_all(t, "//DefNode[.//ReturnStatNode and .//NameNode]")))
+        self.assertEquals(0, len(find_all(t, "//NameNode[@honking and @name]")))
+        self.assertEquals(0, len(find_all(t, "//NameNode[@name and @honking]")))
+        self.assertEquals(2, len(find_all(t, "//DefNode[.//NameNode[@name] and @name]")))
+
+    def test_node_path_attribute_string_predicate(self):
+        t = self._build_tree()
+        self.assertEquals(1, len(find_all(t, "//NameNode[@name = 'decorator']")))
+
+    def test_node_path_recursive_predicate(self):
+        t = self._build_tree()
+        self.assertEquals(2, len(find_all(t, "//DefNode[.//NameNode[@name]]")))
+        self.assertEquals(1, len(find_all(t, "//DefNode[.//NameNode[@name = 'decorator']]")))
+        self.assertEquals(1, len(find_all(t, "//DefNode[.//ReturnStatNode[./NameNode[@name = 'fun']]/NameNode]")))
+
+if __name__ == '__main__':
+    unittest.main()

Cython/Compiler/TreeFragment.py

@@ -3,7 +3,7 @@
 #
 
 import re
-from cStringIO import StringIO
+from StringIO import StringIO
 from Scanning import PyrexScanner, StringSourceDescriptor
 from Symtab import BuiltinScope, ModuleScope
 import Symtab
@@ -54,7 +54,7 @@ def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):
     context = StringParseContext([], name)
     scope = context.find_module(module_name, pos = initial_pos, need_pxd = 0)
 
-    buf = StringIO(code.encode(encoding))
+    buf = StringIO(code)
 
     scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
                      scope = scope, context = context, initial_pos = initial_pos)