merge

Cython/Compiler/Builtin.py

@@ -6,6 +6,7 @@ from Symtab import BuiltinScope, StructOrUnionScope
 from Cython.Utils import UtilityCode
 from TypeSlots import Signature
 import PyrexTypes
+import Naming
 
 builtin_function_table = [
     # name,        args,   return,  C API func,           py equiv = "*"
@@ -16,6 +17,7 @@ builtin_function_table = [
     ('delattr',    "OO",   "r",     "PyObject_DelAttr"),
     ('dir',        "O",    "O",     "PyObject_Dir"),
     ('divmod',     "OO",   "O",     "PyNumber_Divmod"),
+    ('exec',       "OOO",  "O",     "__Pyx_PyRun"),
     #('eval',      "",     "",      ""),
     #('execfile',  "",     "",      ""),
     #('filter',    "",     "",      ""),
@@ -154,6 +156,48 @@ bad:
 }
 """)
 
+pyexec_utility_code = UtilityCode(
+proto = """
+static PyObject* __Pyx_PyRun(PyObject*, PyObject*, PyObject*);
+""",
+impl = """
+static PyObject* __Pyx_PyRun(PyObject* o, PyObject* globals, PyObject* locals) {
+    PyObject* result;
+    PyObject* s = 0;
+
+    if (!locals && !globals) {
+        globals = PyModule_GetDict(%s);""" % Naming.module_cname + """
+        if (!globals)
+            goto bad;
+        locals = globals;
+    } else if (!locals) {
+        locals = globals;
+    } else if (!globals) {
+        globals = locals;
+    }
+
+    if (PyUnicode_Check(o)) {
+        s = PyUnicode_AsUTF8String(o);
+        if (!s) goto bad;
+        o = s;
+    } else if (!PyString_Check(o)) {
+        /* FIXME: support file objects and code objects */
+        PyErr_SetString(PyExc_TypeError,
+            "exec currently requires a string as code input.");
+        goto bad;
+    }
+
+    result = PyRun_String(
+        PyString_AS_STRING(o), Py_file_input, globals, locals);
+
+    Py_XDECREF(s);
+    return result;
+bad:
+    Py_XDECREF(s);
+    return 0;
+}
+""")
+
 intern_utility_code = UtilityCode(
 proto = """
 #if PY_MAJOR_VERSION >= 3
@@ -273,6 +317,7 @@ Py_XDECREF(__Pyx_PyFrozenSet_Type); __Pyx_PyFrozenSet_Type = NULL;
 """)
 
 builtin_utility_code = {
+    'exec'      : pyexec_utility_code,
     'getattr3'  : getattr3_utility_code,
     'intern'    : intern_utility_code,
     'set'       : py23_set_utility_code,

Cython/Compiler/Code.py

@@ -293,7 +293,7 @@ class GlobalState(object):
     def add_interned_num_decl(self, entry):
         if self.should_declare(entry.cname, entry):
             if entry.init[-1] == "L":
-                self.initwriter.putln('%s = PyLong_FromString("%s", 0, 0); %s;' % (
+                self.initwriter.putln('%s = PyLong_FromString((char *)"%s", 0, 0); %s;' % (
                     entry.cname,
                     entry.init,
                     self.initwriter.error_goto_if_null(entry.cname, self.module_pos)))

Cython/Compiler/ExprNodes.py

@@ -899,7 +899,7 @@ class LongNode(AtomicExprNode):
 
     def generate_evaluation_code(self, code):
         code.putln(
-            '%s = PyLong_FromString("%s", 0, 0); %s' % (
+            '%s = PyLong_FromString((char *)"%s", 0, 0); %s' % (
                 self.result(),
                 self.value,
                 code.error_goto_if_null(self.result(), self.pos)))
@@ -1779,19 +1779,29 @@ class SliceIndexNode(ExprNode):
             self.start.analyse_types(env)
         if self.stop:
             self.stop.analyse_types(env)
-        self.base = self.base.coerce_to_pyobject(env)
+        if self.base.type.is_array or self.base.type.is_ptr:
+            # we need a ptr type here instead of an array type, as
+            # array types can result in invalid type casts in the C
+            # code
+            self.type = PyrexTypes.CPtrType(self.base.type.base_type)
+        else:
+            self.base = self.base.coerce_to_pyobject(env)
+            self.type = py_object_type
         c_int = PyrexTypes.c_py_ssize_t_type
         if self.start:
             self.start = self.start.coerce_to(c_int, env)
         if self.stop:
             self.stop = self.stop.coerce_to(c_int, env)
-        self.type = py_object_type
         self.gil_check(env)
         self.is_temp = 1
 
     gil_message = "Slicing Python object"
 
     def generate_result_code(self, code):
+        if not self.type.is_pyobject:
+            error(self.pos,
+                  "Slicing is not currently supported for '%s'." % self.type)
+            return
         code.putln(
             "%s = PySequence_GetSlice(%s, %s, %s); %s" % (
                 self.result(),
@@ -1802,16 +1812,39 @@ class SliceIndexNode(ExprNode):
     
     def generate_assignment_code(self, rhs, code):
         self.generate_subexpr_evaluation_code(code)
-        code.put_error_if_neg(self.pos, 
-            "PySequence_SetSlice(%s, %s, %s, %s)" % (
-                self.base.py_result(),
-                self.start_code(),
-                self.stop_code(),
-                rhs.result()))
+        if self.type.is_pyobject:
+            code.put_error_if_neg(self.pos, 
+                "PySequence_SetSlice(%s, %s, %s, %s)" % (
+                    self.base.py_result(),
+                    self.start_code(),
+                    self.stop_code(),
+                    rhs.result()))
+        else:
+            start_offset = ''
+            if self.start:
+                start_offset = self.start_code()
+                if start_offset == '0':
+                    start_offset = ''
+                else:
+                    start_offset += '+'
+            if rhs.type.is_array:
+                # FIXME: we should check both array sizes here
+                array_length = rhs.type.size
+            else:
+                # FIXME: fix the array size according to start/stop
+                array_length = self.base.type.size
+            for i in range(array_length):
+                code.putln("%s[%s%s] = %s[%d];" % (
+                        self.base.result(), start_offset, i,
+                        rhs.result(), i))
         self.generate_subexpr_disposal_code(code)
         rhs.generate_disposal_code(code)
 
     def generate_deletion_code(self, code):
+        if not self.type.is_pyobject:
+            error(self.pos,
+                  "Deleting slices is only supported for Python types, not '%s'." % self.type)
+            return
         self.generate_subexpr_evaluation_code(code)
         code.put_error_if_neg(self.pos,
             "PySequence_DelSlice(%s, %s, %s)" % (
@@ -1829,6 +1862,8 @@ class SliceIndexNode(ExprNode):
     def stop_code(self):
         if self.stop:
             return self.stop.result()
+        elif self.base.type.is_array:
+            return self.base.type.size
         else:
             return "PY_SSIZE_T_MAX"
     
@@ -4692,8 +4727,8 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list) {
     empty_dict = PyDict_New();
     if (!empty_dict)
         goto bad;
-    module = PyObject_CallFunction(__import__, "OOOO",
-        name, global_dict, empty_dict, list);
+    module = PyObject_CallFunctionObjArgs(__import__,
+        name, global_dict, empty_dict, list, NULL);
 bad:
     Py_XDECREF(empty_list);
     Py_XDECREF(__import__);
@@ -4810,11 +4845,11 @@ static int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) {
 
 create_class_utility_code = UtilityCode(
 proto = """
-static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name, char *modname); /*proto*/
+static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *name, const char *modname); /*proto*/
 """,
 impl = """
 static PyObject *__Pyx_CreateClass(
-    PyObject *bases, PyObject *dict, PyObject *name, char *modname)
+    PyObject *bases, PyObject *dict, PyObject *name, const char *modname)
 {
     PyObject *py_modname;
     PyObject *result = 0;
@@ -4877,7 +4912,12 @@ static INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) {
         return Py_None; // this is just to have an accurate signature
     }
     else {
-        return PyObject_CallMethod(L, "append", "(O)", x);
+        PyObject *r, *m;
+        m = PyObject_GetAttrString(L, "append");
+        if (!m) return NULL;
+        r = PyObject_CallFunctionObjArgs(m, x, NULL);
+        Py_DECREF(m);
+        return r;
     }
 }
 """,
@@ -4968,10 +5008,10 @@ impl = """
 
 raise_noneattr_error_utility_code = UtilityCode(
 proto = """
-static INLINE void __Pyx_RaiseNoneAttributeError(char* attrname);
+static INLINE void __Pyx_RaiseNoneAttributeError(const char* attrname);
 """,
 impl = """
-static INLINE void __Pyx_RaiseNoneAttributeError(char* attrname) {
+static INLINE void __Pyx_RaiseNoneAttributeError(const char* attrname) {
     PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%s'", attrname);
 }
 """)

Cython/Compiler/Lexicon.py

@@ -7,6 +7,7 @@
 
 raw_prefixes = "rR"
 string_prefixes = "cCuUbB"
+IDENT = 'IDENT'
 
 def make_lexicon():
     from Cython.Plex import \
@@ -82,7 +83,7 @@ def make_lexicon():
     comment = Str("#") + Rep(AnyBut("\n"))    
     
     return Lexicon([
-        (name, 'IDENT'),
+        (name, IDENT),
         (intliteral, 'INT'),
         (fltconst, 'FLOAT'),
         (imagconst, 'IMAG'),

Cython/Compiler/Main.py

@@ -102,8 +102,8 @@ class Context:
             NormalizeTree(self),
             PostParse(self),
             _specific_post_parse,
-            _align_function_definitions,
             InterpretCompilerDirectives(self, self.pragma_overrides),
+            _align_function_definitions,
             FlattenInListTransform(),
             WithTransform(self),
             DecoratorTransform(self),

Cython/Compiler/ModuleNode.py

@@ -1448,7 +1448,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     flags = "READONLY"
                 else:
                     flags = "0"
-                code.putln('{"%s", %s, %s, %s, 0},' % (
+                code.putln('{(char *)"%s", %s, %s, %s, 0},' % (
                     entry.name,
                     type_code,
                     "offsetof(%s, %s)" % (objstruct, entry.cname),
@@ -1466,7 +1466,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     env.getset_table_cname)
             for entry in env.property_entries:
                 code.putln(
-                    '{"%s", %s, %s, %s, 0},' % (
+                    '{(char *)"%s", %s, %s, %s, 0},' % (
                         entry.name,
                         entry.getter_cname or "0",
                         entry.setter_cname or "0",
@@ -2050,10 +2050,10 @@ bad:
 
 function_export_utility_code = UtilityCode(
 proto = """
-static int __Pyx_ExportFunction(char *name, void *f, char *sig); /*proto*/
+static int __Pyx_ExportFunction(const char *name, void *f, const char *sig); /*proto*/
 """,
 impl = r"""
-static int __Pyx_ExportFunction(char *name, void *f, char *sig) {
+static int __Pyx_ExportFunction(const char *name, void *f, const char *sig) {
     PyObject *d = 0;
     PyObject *p = 0;
     d = PyObject_GetAttrString(%(MODULE)s, "%(API)s");
@@ -2066,7 +2066,7 @@ static int __Pyx_ExportFunction(char *name, void *f, char *sig) {
         if (PyModule_AddObject(%(MODULE)s, "%(API)s", d) < 0)
             goto bad;
     }
-    p = PyCObject_FromVoidPtrAndDesc(f, sig, 0);
+    p = PyCObject_FromVoidPtrAndDesc(f, (void *)sig, 0);
     if (!p)
         goto bad;
     if (PyDict_SetItemString(d, name, p) < 0)
@@ -2085,16 +2085,16 @@ bad:
 
 function_import_utility_code = UtilityCode(
 proto = """
-static int __Pyx_ImportFunction(PyObject *module, char *funcname, void **f, char *sig); /*proto*/
+static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void **f, const char *sig); /*proto*/
 """,
 impl = """
 #ifndef __PYX_HAVE_RT_ImportFunction
 #define __PYX_HAVE_RT_ImportFunction
-static int __Pyx_ImportFunction(PyObject *module, char *funcname, void **f, char *sig) {
+static int __Pyx_ImportFunction(PyObject *module, const char *funcname, void **f, const char *sig) {
     PyObject *d = 0;
     PyObject *cobj = 0;
     char *desc;
-    
+
     d = PyObject_GetAttrString(module, "%(API)s");
     if (!d)
         goto bad;

Cython/Compiler/Options.py

@@ -60,13 +60,15 @@ option_types = {
     'nonecheck' : bool,
     'embedsignature' : bool,
     'locals' : dict,
+    'auto_cpdef': bool,
 }
 
 option_defaults = {
     'boundscheck' : True,
     'nonecheck' : False,
     'embedsignature' : False,
-    'locals' : {}
+    'locals' : {},
+    'auto_cpdef': False,
 }
 
 def parse_option_value(name, value):

Cython/Compiler/ParseTreeTransforms.py

@@ -234,6 +234,22 @@ class PxdPostParse(CythonTransform):
         if (isinstance(node, DefNode) and self.scope_type == 'cclass'
             and node.name in ('__getbuffer__', '__releasebuffer__')):
             ok = True
+            
+        if isinstance(node, CFuncDefNode):
+            ok = True
+            for stat in node.body.stats:
+                if not isinstance(stat, CVarDefNode):
+                    self.context.error("C function definition not allowed here")
+                    ok = False
+                    break
+            node = CVarDefNode(node.pos, 
+                visibility = node.visibility,
+                base_type = node.base_type,
+                declarators = [node.declarator],
+                in_pxd = True,
+                api = node.api,
+                overridable = node.overridable, 
+                pxd_locals = node.body.stats)
 
         if not ok:
             self.context.nonfatal_error(PostParseError(node.pos,
@@ -547,6 +563,8 @@ property NAME:
                     lenv.declare_var(var, type, type_node.pos)
                 else:
                     error(type_node.pos, "Not a type")
+        for stat in node.pxd_locals:
+            stat.analyse_declarations(lenv)
         node.body.analyse_declarations(lenv)
         self.env_stack.append(lenv)
         self.visitchildren(node)
@@ -601,6 +619,7 @@ class AlignFunctionDefinitions(CythonTransform):
     
     def visit_ModuleNode(self, node):
         self.scope = node.scope
+        self.directives = node.directives
         self.visitchildren(node)
         return node
     
@@ -613,8 +632,8 @@ class AlignFunctionDefinitions(CythonTransform):
                 error(node.pos, "'%s' redeclared" % node.name)
                 error(pxd_def.pos, "previous declaration here")
                 return None
-        self.visitchildren(node)
-        return node
+        else:
+            return node
         
     def visit_CClassDefNode(self, node, pxd_def=None):
         if pxd_def is None:
@@ -636,6 +655,8 @@ class AlignFunctionDefinitions(CythonTransform):
                 error(node.pos, "'%s' redeclared" % node.name)
                 error(pxd_def.pos, "previous declaration here")
                 return None
+        elif self.scope.is_module_scope and self.directives['auto_cpdef']:
+            node = node.as_cfunction(scope=self.scope)
         # Enable this when internal def functions are allowed. 
         # self.visitchildren(node)
         return node

Cython/Compiler/Parsing.pxd

+# We declare all of these here to type the first argument.
+
+from Cython.Compiler.Scanning cimport PyrexScanner
+
+
+cpdef p_ident(PyrexScanner s, message =*)
+cpdef p_ident_list(PyrexScanner s)
+
+cpdef p_binop_expr(PyrexScanner s, ops, p_sub_expr)
+cpdef p_simple_expr(PyrexScanner s)
+cpdef p_test(PyrexScanner s)
+cpdef p_or_test(PyrexScanner s)
+cpdef p_rassoc_binop_expr(PyrexScanner s, ops, p_subexpr)
+cpdef p_and_test(PyrexScanner s)
+cpdef p_not_test(PyrexScanner s)
+cpdef p_comparison(PyrexScanner s)
+cpdef p_cascaded_cmp(PyrexScanner s)
+cpdef p_cmp_op(PyrexScanner s)
+cpdef p_bit_expr(PyrexScanner s)
+cpdef p_xor_expr(PyrexScanner s)
+cpdef p_and_expr(PyrexScanner s)
+cpdef p_shift_expr(PyrexScanner s)
+cpdef p_arith_expr(PyrexScanner s)
+cpdef p_term(PyrexScanner s)
+cpdef p_factor(PyrexScanner s)
+cpdef p_typecast(PyrexScanner s)
+cpdef p_sizeof(PyrexScanner s)
+cpdef p_power(PyrexScanner s)
+cpdef p_trailer(PyrexScanner s, node1)
+cpdef p_call(PyrexScanner s, function)
+cpdef p_index(PyrexScanner s, base)
+cpdef p_subscript_list(PyrexScanner s)
+cpdef p_subscript(PyrexScanner s)
+cpdef p_slice_element(PyrexScanner s, follow_set)
+cpdef expect_ellipsis(PyrexScanner s)
+cpdef make_slice_nodes(pos, subscripts)
+cpdef make_slice_node(pos, start, stop = *, step = *)
+cpdef p_atom(PyrexScanner s)
+cpdef p_name(PyrexScanner s, name)
+cpdef p_cat_string_literal(PyrexScanner s)
+cpdef p_opt_string_literal(PyrexScanner s)
+cpdef p_string_literal(PyrexScanner s)
+cpdef p_list_maker(PyrexScanner s)
+cpdef p_list_iter(PyrexScanner s)
+cpdef p_list_for(PyrexScanner s)
+cpdef p_list_if(PyrexScanner s)
+cpdef p_dict_maker(PyrexScanner s)
+cpdef p_dict_item(PyrexScanner s)
+cpdef p_backquote_expr(PyrexScanner s)
+cpdef p_simple_expr_list(PyrexScanner s)
+cpdef p_expr(PyrexScanner s)
+cpdef p_testlist(PyrexScanner s)
+
+#-------------------------------------------------------
+#
+#   Statements
+#
+#-------------------------------------------------------
+
+cpdef p_global_statement(PyrexScanner s)
+cpdef p_expression_or_assignment(PyrexScanner s)
+cpdef p_print_statement(PyrexScanner s)
+cpdef p_del_statement(PyrexScanner s)
+cpdef p_pass_statement(PyrexScanner s, bint with_newline = *)
+cpdef p_break_statement(PyrexScanner s)
+cpdef p_continue_statement(PyrexScanner s)
+cpdef p_return_statement(PyrexScanner s)
+cpdef p_raise_statement(PyrexScanner s)
+cpdef p_import_statement(PyrexScanner s)
+cpdef p_from_import_statement(PyrexScanner s, bint first_statement = *)
+cpdef p_imported_name(PyrexScanner s, bint is_cimport)
+cpdef p_dotted_name(PyrexScanner s, bint as_allowed)
+cpdef p_as_name(PyrexScanner s)
+cpdef p_assert_statement(PyrexScanner s)
+cpdef p_if_statement(PyrexScanner s)
+cpdef p_if_clause(PyrexScanner s)
+cpdef p_else_clause(PyrexScanner s)
+cpdef p_while_statement(PyrexScanner s)
+cpdef p_for_statement(PyrexScanner s)
+cpdef p_for_bounds(PyrexScanner s)
+cpdef p_for_from_relation(PyrexScanner s)
+cpdef p_for_from_step(PyrexScanner s)
+cpdef p_target(PyrexScanner s, terminator)
+cpdef p_for_target(PyrexScanner s)
+cpdef p_for_iterator(PyrexScanner s)
+cpdef p_try_statement(PyrexScanner s)
+cpdef p_except_clause(PyrexScanner s)
+cpdef p_include_statement(PyrexScanner s, ctx)
+cpdef p_with_statement(PyrexScanner s)
+cpdef p_simple_statement(PyrexScanner s, bint first_statement = *)
+cpdef p_simple_statement_list(PyrexScanner s, ctx, bint first_statement = *)
+cpdef p_compile_time_expr(PyrexScanner s)
+cpdef p_DEF_statement(PyrexScanner s)
+cpdef p_IF_statement(PyrexScanner s, ctx)
+cpdef p_statement(PyrexScanner s, ctx, bint first_statement = *)
+cpdef p_statement_list(PyrexScanner s, ctx, bint first_statement = *)
+cpdef p_suite(PyrexScanner s, ctx = *, bint with_doc = *, bint with_pseudo_doc = *)
+cpdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, type_positions= *, type_keywords= * )
+
+cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *)
+cpdef p_calling_convention(PyrexScanner s)
+cpdef p_c_complex_base_type(PyrexScanner s)
+cpdef p_c_simple_base_type(PyrexScanner s, self_flag, nonempty)
+cpdef p_buffer_access(PyrexScanner s, base_type_node)
+cpdef bint looking_at_name(PyrexScanner s) except -2
+cpdef bint looking_at_expr(PyrexScanner s) except -2
+cpdef bint looking_at_base_type(PyrexScanner s) except -2
+cpdef bint looking_at_dotted_name(PyrexScanner s) except -2
+cpdef p_sign_and_longness(PyrexScanner s)
+cpdef p_opt_cname(PyrexScanner s)
+cpdef p_c_declarator(PyrexScanner s, ctx = *, bint empty = *, bint is_type = *, bint cmethod_flag = *,
+                   bint assignable = *, bint nonempty = *,
+                   bint calling_convention_allowed = *)
+cpdef p_c_array_declarator(PyrexScanner s, base)
+cpdef p_c_func_declarator(PyrexScanner s, pos, ctx, base, bint cmethod_flag)
+cpdef p_c_simple_declarator(PyrexScanner s, ctx, bint empty, bint is_type, bint cmethod_flag,
+                          bint assignable, bint nonempty)
+cpdef p_nogil(PyrexScanner s)
+cpdef p_with_gil(PyrexScanner s)
+cpdef p_exception_value_clause(PyrexScanner s)
+cpdef p_c_arg_list(PyrexScanner s, ctx = *, bint in_pyfunc = *, bint cmethod_flag = *,
+                 bint nonempty_declarators = *, bint kw_only = *)
+cpdef p_optional_ellipsis(PyrexScanner s)
+cpdef p_c_arg_decl(PyrexScanner s, ctx, in_pyfunc, bint cmethod_flag = *, bint nonempty = *, bint kw_only = *)
+cpdef p_api(PyrexScanner s)
+cpdef p_cdef_statement(PyrexScanner s, ctx)
+cpdef p_cdef_block(PyrexScanner s, ctx)
+cpdef p_cdef_extern_block(PyrexScanner s, pos, ctx)
+cpdef p_c_enum_definition(PyrexScanner s, pos, ctx)
+cpdef p_c_enum_line(PyrexScanner s, items)
+cpdef p_c_enum_item(PyrexScanner s, items)
+cpdef p_c_struct_or_union_definition(PyrexScanner s, pos, ctx)
+cpdef p_visibility(PyrexScanner s, prev_visibility)
+cpdef p_c_modifiers(PyrexScanner s)
+cpdef p_c_func_or_var_declaration(PyrexScanner s, pos, ctx)
+cpdef p_ctypedef_statement(PyrexScanner s, ctx)
+cpdef p_decorators(PyrexScanner s)
+cpdef p_def_statement(PyrexScanner s, decorators = *)
+cpdef p_py_arg_decl(PyrexScanner s)
+cpdef p_class_statement(PyrexScanner s)
+cpdef p_c_class_definition(PyrexScanner s, pos,  ctx)
+cpdef p_c_class_options(PyrexScanner s)
+cpdef p_property_decl(PyrexScanner s)
+cpdef p_doc_string(PyrexScanner s)
+cpdef p_code(PyrexScanner s, level= *)
+cpdef p_compiler_directive_comments(PyrexScanner s)
+cpdef p_module(PyrexScanner s, pxd, full_module_name)

Cython/Compiler/Parsing.py

+# cython: auto_cpdef=True
 #
 #   Pyrex Parser
 #
 
+# This should be done automatically
+import cython
+cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)
+
 import os
 import re
 import sys
 from types import ListType, TupleType
-from Scanning import PyrexScanner, FileSourceDescriptor
+from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor
 import Nodes
 import ExprNodes
 import StringEncoding
@@ -527,26 +532,22 @@ def p_atom(s):
 
 def p_name(s, name):
     pos = s.position()
-    if not s.compile_time_expr:
-        try:
-            value = s.compile_time_env.lookup_here(name)
-        except KeyError:
-            pass
+    if not s.compile_time_expr and name in s.compile_time_env:
+        value = s.compile_time_env.lookup_here(name)
+        rep = repr(value)
+        if isinstance(value, bool):
+            return ExprNodes.BoolNode(pos, value = value)
+        elif isinstance(value, int):
+            return ExprNodes.IntNode(pos, value = rep)
+        elif isinstance(value, long):
+            return ExprNodes.IntNode(pos, value = rep, longness = "L")
+        elif isinstance(value, float):
+            return ExprNodes.FloatNode(pos, value = rep)
+        elif isinstance(value, (str, unicode)):
+            return ExprNodes.StringNode(pos, value = value)
         else:
-            rep = repr(value)
-            if isinstance(value, bool):
-                return ExprNodes.BoolNode(pos, value = value)
-            elif isinstance(value, int):
-                return ExprNodes.IntNode(pos, value = rep)
-            elif isinstance(value, long):
-                return ExprNodes.IntNode(pos, value = rep, longness = "L")
-            elif isinstance(value, float):
-                return ExprNodes.FloatNode(pos, value = rep)
-            elif isinstance(value, (str, unicode)):
-                return ExprNodes.StringNode(pos, value = value)
-            else:
-                error(pos, "Invalid type for compile-time constant: %s"
-                    % value.__class__.__name__)
+            error(pos, "Invalid type for compile-time constant: %s"
+                % value.__class__.__name__)
     return ExprNodes.NameNode(pos, name = name)
 
 def p_cat_string_literal(s):
@@ -902,6 +903,21 @@ def p_print_statement(s):
     return Nodes.PrintStatNode(pos,
         arg_tuple = arg_tuple, append_newline = not ends_with_comma)
 
+def p_exec_statement(s):
+    # s.sy == 'exec'
+    pos = s.position()
+    s.next()
+    args = [ p_bit_expr(s) ]
+    if s.sy == 'in':
+        s.next()
+        args.append(p_simple_expr(s))
+        if s.sy == ',':
+            s.next()
+            args.append(p_simple_expr(s))
+    else:
+        error(pos, "'exec' currently requires a target mapping (globals/locals)")
+    return Nodes.ExecStatNode(pos, args = args)
+
 def p_del_statement(s):
     # s.sy == 'del'
     pos = s.position()
@@ -1307,7 +1323,7 @@ def p_with_statement(s):
             if not allow_multi and isinstance(target, ExprNodes.TupleNode):
                 s.error("Multiple with statement target values not allowed without paranthesis")
         body = p_suite(s)
-	return Nodes.WithStatNode(pos, manager = manager, 
+    return Nodes.WithStatNode(pos, manager = manager, 
 	       			       target = target, body = body)
     
 def p_simple_statement(s, first_statement = 0):
@@ -1316,6 +1332,8 @@ def p_simple_statement(s, first_statement = 0):
         node = p_global_statement(s)
     elif s.sy == 'print':
         node = p_print_statement(s)
+    elif s.sy == 'exec':
+        node = p_exec_statement(s)
     elif s.sy == 'del':
         node = p_del_statement(s)
     elif s.sy == 'break':
@@ -2128,7 +2146,7 @@ def p_c_func_or_var_declaration(s, pos, ctx):
                                 assignable = 1, nonempty = 1)
     declarator.overridable = ctx.overridable
     if s.sy == ':':
-        if ctx.level not in ('module', 'c_class'):
+        if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):
             s.error("C function definition not allowed here")
         doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
         result = Nodes.CFuncDefNode(pos,
@@ -2382,7 +2400,7 @@ def p_code(s, level=None):
             repr(s.sy), repr(s.systring)))
     return body
 
-COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*([a-z]+)\s*=(.*)$")
+COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*([a-z_]+)\s*=(.*)$")
 
 def p_compiler_directive_comments(s):
     result = {}

Cython/Compiler/Scanning.pxd

+from Cython.Plex.Scanners cimport Scanner
+
+cdef class CompileTimeScope:
+    cdef public entries
+    cdef public outer
+
+cdef class PyrexScanner(Scanner):
+    cdef public context
+    cdef public list included_files
+    cdef public compile_time_env
+    cdef public bint compile_time_eval
+    cdef public bint compile_time_expr
+    cdef public bint parse_comments
+    cdef public source_encoding
+    cdef public list indentation_stack
+    cdef public indentation_char
+    cdef public int bracket_nesting_level
+    cdef public sy
+    cdef public systring
+
+    cdef long current_level(self)
+    cpdef begin(self, state)
+    cpdef next(self)
+    cpdef bint expect(self, what, message = *) except -2
+    
+    cpdef indentation_action(self, text):
+        cdef:
+            long current_level
+            long new_level
+    

Cython/Compiler/Scanning.py

@@ -11,15 +11,21 @@ import stat
 import sys
 from time import time
 
+import cython
+cython.declare(EncodedString=object, string_prefixes=object, raw_prefixes=object, IDENT=object)
+
 from Cython import Plex, Utils
-from Cython.Plex import Scanner
+from Cython.Plex.Scanners import Scanner
 from Cython.Plex.Errors import UnrecognizedInput
 from Errors import CompileError, error
-from Lexicon import string_prefixes, raw_prefixes, make_lexicon
+from Lexicon import string_prefixes, raw_prefixes, make_lexicon, IDENT
 
 from StringEncoding import EncodedString
 
-plex_version = getattr(Plex, '_version', None)
+try:
+    plex_version = Plex._version
+except AttributeError:
+    plex_version = None
 #print "Plex version:", plex_version ###
 
 debug_scanner = 0
@@ -91,7 +97,7 @@ def try_to_unpickle_lexicon():
     source_file = os.path.join(dir, "Lexicon.py")
     lexicon_hash = hash_source_file(source_file)
     lexicon_pickle = os.path.join(dir, "Lexicon.pickle")
-    f = open_pickled_lexicon(expected_hash = lexicon_hash)
+    f = open_pickled_lexicon(lexicon_hash)
     if f:
         if notify_lexicon_unpickling:
             t0 = time()
@@ -165,9 +171,12 @@ def build_resword_dict():
         d[word] = 1
     return d
 
+cython.declare(resword_dict=object)
+resword_dict = build_resword_dict()
+
 #------------------------------------------------------------------
 
-class CompileTimeScope(object):
+class CompileTimeScope:
 
     def __init__(self, outer = None):
         self.entries = {}
@@ -178,6 +187,9 @@ class CompileTimeScope(object):
     
     def lookup_here(self, name):
         return self.entries[name]
+        
+    def __contains__(self, name):
+        return name in self.entries
     
     def lookup(self, name):
         try:
@@ -286,7 +298,6 @@ class PyrexScanner(Scanner):
     #  compile_time_env   dict     Environment for conditional compilation
     #  compile_time_eval  boolean  In a true conditional compilation context
     #  compile_time_expr  boolean  In a compile-time expression context
-    resword_dict = build_resword_dict()
 
     def __init__(self, file, filename, parent_scanner = None, 
                  scope = None, context = None, source_encoding=None, parse_comments=True, initial_pos=None):
@@ -403,15 +414,15 @@ class PyrexScanner(Scanner):
             sy, systring = self.read()
         except UnrecognizedInput:
             self.error("Unrecognized character")
-        if sy == 'IDENT':
-            if systring in self.resword_dict:
+        if sy == IDENT:
+            if systring in resword_dict:
                 sy = systring
             else:
                 systring = EncodedString(systring)
                 systring.encoding = self.source_encoding
         self.sy = sy
         self.systring = systring
-        if debug_scanner:
+        if False: # debug_scanner:
             _, line, col = self.position()
             if not self.systring or self.sy == self.systring:
                 t = self.sy
@@ -443,7 +454,7 @@ class PyrexScanner(Scanner):
             self.expected(what, message)
     
     def expect_keyword(self, what, message = None):
-        if self.sy == 'IDENT' and self.systring == what:
+        if self.sy == IDENT and self.systring == what:
             self.next()
         else:
             self.expected(what, message)

Cython/Compiler/Symtab.py

@@ -821,6 +821,8 @@ class ModuleScope(Scope):
         self.cached_builtins = []
         self.undeclared_cached_builtins = []
         self.namespace_cname = self.module_cname
+        for name in ['__builtins__', '__name__', '__file__', '__doc__']:
+            self.declare_var(EncodedString(name), py_object_type, None)
     
     def qualifying_scope(self):
         return self.parent_module

Cython/Compiler/Visitor.py

@@ -14,26 +14,33 @@ class BasicVisitor(object):
     # instance.
     def __init__(self):
         self.dispatch_table = {}
-     
+
     def visit(self, obj):
-        cls = obj.__class__
-        m = self.dispatch_table.get(cls.__name__)
-        if m is None:
+        cls = type(obj)
+        try:
+            handler_method = self.dispatch_table[cls]
+        except KeyError:
+            #print "Cache miss for class %s in visitor %s" % (
+            #    cls.__name__, type(self).__name__)
             # Must resolve, try entire hierarchy
             pattern = "visit_%s"
             mro = inspect.getmro(cls)
-            for cls in mro:
-                m = getattr(self, pattern % cls.__name__, None)
-                if m is not None:
+            for mro_cls in mro:
+                try:
+                    handler_method = getattr(self, pattern % mro_cls.__name__)
                     break
+                except AttributeError:
+                    pass
             else:
                 print type(self), type(obj)
-                print self.access_path
-                print self.access_path[-1][0].pos
-                print self.access_path[-1][0].__dict__
+                if hasattr(self, 'access_path'):
+                    print self.access_path
+                    print self.access_path[-1][0].pos
+                    print self.access_path[-1][0].__dict__
                 raise RuntimeError("Visitor does not accept object: %s" % obj)
-            self.dispatch_table[cls.__name__] = m
-        return m(obj)
+            #print "Caching " + cls.__name__
+            self.dispatch_table[cls] = handler_method
+        return handler_method(obj)
 
 class TreeVisitor(BasicVisitor):
     """

Cython/Plex/Scanners.pxd

+import cython
+
+cdef class Scanner:
+
+    cdef public lexicon
+    cdef public stream
+    cdef public name
+    cdef public buffer
+    cdef public long buf_start_pos
+    cdef public long next_pos
+    cdef public long cur_pos
+    cdef public long cur_line
+    cdef public long cur_line_start
+    cdef public long start_pos
+    cdef public long start_line
+    cdef public long start_col
+    cdef public text
+    cdef public initial_state # int?
+    cdef public state_name
+    cdef public list queue
+    cdef public bint trace
+    cdef public cur_char
+    cdef public input_state
+
+    cdef public level
+    
+    cpdef next_char(self):
+        cdef:
+            long input_state
+    cpdef read(self)
+    cpdef position(self)
+    
+    cpdef run_machine_inlined(self):
+        cdef:
+            long cur_pos
+            long cur_line
+            long cur_line_start
+            long input_state
+            long next_pos
+            long buf_start_pos
+            long buf_len
+            long buf_index
+            bint trace
+            long discard
+    
+    cpdef begin(self, state)
+    cpdef produce(self, value, text = *)

Cython/Plex/Scanners.py

@@ -10,6 +10,8 @@
 import Errors
 from Regexps import BOL, EOL, EOF
 
+import cython
+
 class Scanner:
   """
   A Scanner is used to read tokens from a stream of characters
@@ -42,23 +44,23 @@ class Scanner:
 
   """
 
-  lexicon = None        # Lexicon
-  stream = None         # file-like object
-  name = ''
-  buffer = ''
-  buf_start_pos = 0     # position in input of start of buffer
-  next_pos = 0          # position in input of next char to read
-  cur_pos = 0           # position in input of current char
-  cur_line = 1          # line number of current char
-  cur_line_start = 0    # position in input of start of current line
-  start_pos = 0         # position in input of start of token
-  start_line = 0        # line number of start of token
-  start_col = 0         # position in line of start of token
-  text = None           # text of last token read
-  initial_state = None  # Node
-  state_name = ''       # Name of initial state
-  queue = None          # list of tokens to be returned
-  trace = 0
+#  lexicon = None        # Lexicon
+#  stream = None         # file-like object
+#  name = ''
+#  buffer = ''
+#  buf_start_pos = 0     # position in input of start of buffer
+#  next_pos = 0          # position in input of next char to read
+#  cur_pos = 0           # position in input of current char
+#  cur_line = 1          # line number of current char
+#  cur_line_start = 0    # position in input of start of current line
+#  start_pos = 0         # position in input of start of token
+#  start_line = 0        # line number of start of token
+#  start_col = 0         # position in line of start of token
+#  text = None           # text of last token read
+#  initial_state = None  # Node
+#  state_name = ''       # Name of initial state
+#  queue = None          # list of tokens to be returned
+#  trace = 0
 
   def __init__(self, lexicon, stream, name = '', initial_pos = None):
     """
@@ -73,6 +75,17 @@ class Scanner:
       |name| is optional, and may be the name of the file being
       scanned or any other identifying string.
     """
+    self.buffer = ''
+    self.buf_start_pos = 0
+    self.next_pos = 0
+    self.cur_pos = 0
+    self.cur_line = 1
+    self.start_pos = 0
+    self.start_line = 0
+    self.start_col = 0
+    self.text = None
+    self.state_name = None
+    
     self.lexicon = lexicon
     self.stream = stream
     self.name = name
@@ -374,6 +387,3 @@ class Scanner:
     Override this method if you want something to be done at
     end of file.
     """
-
-# For backward compatibility:
-setattr(Scanner, "yield", Scanner.produce)

MANIFEST.in

-include MANIFEST.in README.txt INSTALL.txt CHANGES.txt ToDo.txt USAGE.txt
-include CHANGES_pyrex.txt COPYING.txt LICENSE.txt Makefile
+include MANIFEST.in README.txt INSTALL.txt ToDo.txt USAGE.txt
+include COPYING.txt LICENSE.txt Makefile
 recursive-include .hg *
 include .hgignore .hgtags
 include setup.py
-include bin/cython bin/update_references
+include bin/cython
 include cython.py
 include Cython/Compiler/Lexicon.pickle
 include Cython/Includes/*.pxd
 include Doc/*
 
 include Demos/*.pyx
-include Demos/*.pxd
 include Demos/*.py
 include Demos/callback/*
 include Demos/embed/*

ToDo.txt

+See http://trac.cython.org/cython_trac and http://wiki.cython.org/enhancements
+
+
 -- The Original Pyrex Todo List --
 
 DONE - Pointer-to-function types.

pyximport/pyximport.py

@@ -20,12 +20,17 @@ the documentation.
 This code was modeled on Quixote's ptl_import.
 """
 import sys, os, shutil
-import imp, ihooks, glob, md5
+import imp, ihooks, glob
 import __builtin__
 import pyxbuild
 from distutils.dep_util import newer
 from distutils.extension import Extension
 
+try:
+    import hashlib
+except ImportError:
+    import md5 as hashlib
+
 mod_name = "pyximport"
 
 assert sys.hexversion >= 0x20000b1, "need Python 2.0b1 or later"
@@ -54,7 +59,7 @@ def _load_pyrex(name, filename):
 
 def get_distutils_extension(modname, pyxfilename):
 
-    extra = "_" + md5.md5(open(pyxfilename).read()).hexdigest()  
+    extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()  
 #    modname = modname + extra
 
     extension_mod = handle_special_build(modname, pyxfilename)

setup.py

@@ -31,7 +31,10 @@ except ValueError:
     try:
         from Cython.Compiler.Main import compile
         source_root = os.path.dirname(__file__)
-        compiled_modules = ["Cython.Plex.Scanners"]
+        compiled_modules = ["Cython.Plex.Scanners",
+                            "Cython.Compiler.Scanning",
+                            "Cython.Compiler.Parsing",
+                            "Cython.Compiler.Visitor"]
         extensions = []
         for module in compiled_modules:
             source_file = os.path.join(source_root, *module.split('.'))

tests/run/arrayassign.pyx

+__doc__ = u"""
+>>> test_literal_list_slice_all()
+(1, 2, 3, 4, 5)
+>>> test_literal_list_slice_start()
+(1, 2, 3, 4, 5)
+>>> test_literal_list_slice_end()
+(1, 2, 3, 4, 5)
+>>> test_literal_list_slice_start_end()
+(1, 2, 3, 4, 5)
+
+>>> test_literal_list_slice_start_param(2)
+(1, 2, 3, 4, 5)
+>>> test_literal_list_slice_end_param(4)
+(1, 2, 3, 4, 5)
+>>> test_literal_list_slice_start_end_param(2,7)
+(1, 2, 3, 4, 5)
+
+>>> test_ptr_literal_list_slice_all()
+(1, 2, 3, 4, 5)
+>>> test_ptr_literal_list_slice_start()
+(1, 2, 3, 4, 5)
+>>> test_ptr_literal_list_slice_end()
+(1, 2, 3, 4, 5)
+"""
+
+# this doesn't work - it would reassign the array address!
+#
+#def test_literal_list():
+#    cdef int a[5]
+#    a = [1,2,3,4,5]
+#    return (a[0], a[1], a[2], a[3], a[4])
+
+def test_literal_list_slice_all():
+    cdef int a[5] # = [5,4,3,2,1]
+    a[:] = [1,2,3,4,5]
+    return (a[0], a[1], a[2], a[3], a[4])
+
+def test_literal_list_slice_start():
+    cdef int a[7] # = [7,6,5,4,3,2,1]
+    a[2:] = [1,2,3,4,5]
+    return (a[2], a[3], a[4], a[5], a[6])
+
+def test_literal_list_slice_end():
+    cdef int a[7] # = [7,6,5,4,3,2,1]
+    a[:5] = [1,2,3,4,5]
+    return (a[0], a[1], a[2], a[3], a[4])
+
+def test_literal_list_slice_start_end():
+    cdef int a[9] # = [9,8,7,6,5,4,3,2,1]
+    a[2:7] = [1,2,3,4,5]
+    return (a[2], a[3], a[4], a[5], a[6])
+
+def test_literal_list_slice_start_param(s):
+    cdef int a[9] # = [9,8,7,6,5,4,3,2,1]
+    a[s:] = [1,2,3,4,5]
+    return (a[2], a[3], a[4], a[5], a[6])
+#    return a[s:]
+
+def test_literal_list_slice_end_param(e):
+    cdef int a[9] # = [9,8,7,6,5,4,3,2,1]
+    a[:e] = [1,2,3,4,5]
+    return (a[0], a[1], a[2], a[3], a[4])
+#    return a[:e]
+
+def test_literal_list_slice_start_end_param(s,e):
+    cdef int a[9] # = [9,8,7,6,5,4,3,2,1]
+    a[s:e] = [1,2,3,4,5]
+    return (a[2], a[3], a[4], a[5], a[6])
+#    return a[s:e]
+
+def test_ptr_literal_list_slice_all():
+    cdef int *a = [6,5,4,3,2]
+    a[:] = [1,2,3,4,5]
+    return (a[0], a[1], a[2], a[3], a[4])
+
+def test_ptr_literal_list_slice_start():
+    cdef int *a = [6,5,4,3,2,1]
+    a[1:] = [1,2,3,4,5]
+    return (a[1], a[2], a[3], a[4], a[5])
+
+def test_ptr_literal_list_slice_end():
+    cdef int *a = [6,5,4,3,2,1]
+    a[:5] = [1,2,3,4,5]
+    return (a[0], a[1], a[2], a[3], a[4])
+
+# tuples aren't supported (yet)
+#
+#def test_literal_tuple():
+#    cdef int a[5]
+#    a = (1,2,3,4,5)
+#    return (a[0], a[1], a[2], a[3], a[4])
+
+# this would be nice to have:
+#
+#def test_list(list l):
+#    cdef int a[5]
+#    a[:] = l
+#    return (a[0], a[1], a[2], a[3], a[4])

tests/run/exectest.pyx

+__doc__ = """# no unicode string, not tested in Python3!
+#>>> a
+#Traceback (most recent call last):
+#NameError: name 'a' is not defined
+#>>> test_module_scope()
+#>>> a
+
+>>> test_dict_scope1()
+2
+
+>>> d = {}
+>>> test_dict_scope2(d)
+>>> print d['b']
+2
+
+>>> d1 = {}
+>>> test_dict_scope3(d1, d1)
+>>> print d1['b']
+2
+
+>>> d1, d2 = {}, {}
+>>> test_dict_scope3(d1, d2)
+>>> print d1.get('b'), d2.get('b')
+None 2
+
+>>> d1, d2 = {}, {}
+>>> test_dict_scope3(d1, d2)
+>>> print d1.get('b'), d2.get('b')
+None 2
+
+>>> d1, d2 = dict(a=11), dict(c=5)
+>>> test_dict_scope_ref(d1, d2)
+>>> print d1.get('b'), d2.get('b')
+None 16
+
+>>> d = dict(a=11, c=5)
+>>> test_dict_scope_ref(d, d)
+>>> print d['b']
+16
+
+>>> d = dict(seq = [1,2,3,4])
+>>> add_iter = test_def(d, 'seq')
+>>> list(add_iter())
+[2, 3, 4, 5]
+
+>>> # errors
+
+>>> d1, d2 = {}, {}
+>>> test_dict_scope_ref(d1, d2)
+Traceback (most recent call last):
+NameError: name 'a' is not defined
+"""
+
+#def test_module_scope():
+#    exec "a=1+1"
+#    return __dict__['a']
+
+def test_dict_scope1():
+    cdef dict d = {}
+    exec "b=1+1" in d
+    return d['b']
+
+def test_dict_scope2(d):
+    exec "b=1+1" in d
+
+def test_dict_scope3(d1, d2):
+    exec "b=1+1" in d1, d2
+
+def test_dict_scope_ref(d1, d2):
+    exec "b=a+c" in d1, d2
+
+def test_def(d, varref):
+    exec """
+def test():
+    for x in %s:
+        yield x+1
+""" % varref in d
+    return d['test']

tests/run/importas.pyx

+__doc__ = u"""
+>>> import sys as sous
+>>> import distutils.core as corey
+>>> from copy import deepcopy as copey
+
+>>> sous is _sous
+True
+>>> corey is _corey
+True
+>>> copey is _copey
+True
+
+>>> _sous is not None
+True
+>>> _corey is not None
+True
+>>> _copey is not None
+True
+
+>>> print(_sous.__name__)
+sys
+>>> print(sous.__name__)
+sys
+>>> print(_corey.__name__)
+distutils.core
+>>> print(corey.__name__)
+distutils.core
+>>> print(_copey.__name__)
+deepcopy
+>>> print(copey.__name__)
+deepcopy
+"""
+
+import sys as _sous
+import distutils.core as _corey
+from copy import deepcopy as _copey

tests/run/switch.pyx

@@ -89,6 +89,9 @@ __doc__ = u"""
 1
 >>> switch_off(2)
 0
+
+>>> switch_pass(1)
+1
 """
 
 def switch_simple_py(x):
@@ -173,3 +176,12 @@ def switch_off(int x):
     else:
         return 0
     return -1
+
+def switch_pass(int x):
+    if x == 1:
+        pass
+    elif x == 2:
+        pass
+    else:
+        pass
+    return x