Merge branch 'master' into static

Conflicts:
	CHANGES.rst

.travis.yml

@@ -6,8 +6,11 @@ python:
   - 3.2
   - 3.3
   - 3.4
+  - pypy
 
-#  - pypy
+env:
+  - BACKEND=c
+  - BACKEND=cpp
 
 branches:
   only:
@@ -21,9 +24,14 @@ before_install:
 install: CFLAGS="-O2 -ggdb" pip install .
 
 script:
-  - PYTHON="python$( python -c 'import sys; print("%d.%d" % sys.version_info[:2])' )-dbg"
-  - CFLAGS="-O0 -ggdb" $( $PYTHON -V >&2 && echo $PYTHON || echo python ) runtests.py -vv
+  - PYTHON_DBG="python$( python -c 'import sys; print("%d.%d" % sys.version_info[:2])' )-dbg"
+  - if $PYTHON_DBG -V >&2; then CFLAGS="-O0 -ggdb" $PYTHON_DBG runtests.py -vv Debugger --backends=$BACKEND; fi
+  - CFLAGS="-O0 -ggdb" python runtests.py -vv -x Debugger --backends=$BACKEND
 
 matrix:
   allow_failures:
     - python: pypy
+  exclude:
+    - python: pypy
+      env: BACKEND=cpp
+  fast_finish: true

CHANGES.rst

@@ -9,10 +9,20 @@ Latest
 Features added
 --------------
 
+* Enums can now be declared as cpdef to export their values to
+  the module's Python namespace.  Cpdef enums in pxd files export
+  their values to their own module, iff it exists.
+
 * Allow @staticmethod decorator to declare static cdef methods.
   This is especially useful for declaring "constructors" for
   cdef classes that can take non-Python arguments.
 
+* Taking a ``char*`` from a temporary Python string object is safer
+  in more cases and can be done inside of non-trivial expressions,
+  including arguments of a function call.  A compile time error
+  is raised only when such a pointer is assigned to a variable and
+  would thus exceed the lifetime of the string itself.
+
 * Generators have new properties ``__name__`` and ``__qualname__``
   that provide the plain/qualified name of the generator function
   (following CPython 3.5).  See http://bugs.python.org/issue21205
@@ -28,9 +38,6 @@ Features added
 * HTML output of annotated code uses Pygments for code highlighting
   and generally received a major overhaul by Matthias Bussonier.
 
-* The Python expression "2 ** N" is optimised into bit shifting.
-  See http://bugs.python.org/issue21420
-
 * Simple support for declaring Python object types in Python signature
   annotations.  Currently requires setting the compiler directive
   ``annotation_typing=True``.
@@ -38,9 +45,23 @@ Features added
 * New directive ``use_switch`` (defaults to True) to optionally disable
   the optimization of chained if statement to C switch statements.
 
+Optimizations
+-------------
+
+* The Python expression "2 ** N" is optimised into bit shifting.
+  See http://bugs.python.org/issue21420
+
+* Cascaded assignments (a = b = ...) try to minimise the number of
+  type coercions.
+
+* Calls to ``slice()`` are translated to a straight C-API call.
+
 Bugs fixed
 ----------
 
+* Taking a ``char*`` from an indexed Python string generated unsafe
+  reference counting code.
+
 * Set literals now create all of their items before trying to add them
   to the set, following the behaviour in CPython.  This makes a
   difference in the rare case that the item creation has side effects
@@ -68,7 +89,7 @@ Bugs fixed
 * Syntax highlighting in ``cython-mode.el`` for Emacs no longer
   incorrectly highlights keywords found as part of longer names.
 
-* Correctly handle `from cython.submodule comport name``.
+* Correctly handle ``from cython.submodule cimport name``.
 
 Other changes
 -------------

Cython/Build/Dependencies.py

@@ -792,6 +792,8 @@ def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, fo
         if not os.path.exists(options.cache):
             os.makedirs(options.cache)
     to_compile.sort()
+    if len(to_compile) <= 1:
+        nthreads = 0
     if nthreads:
         # Requires multiprocessing (or Python >= 2.6)
         try:

Cython/Compiler/ExprNodes.py

@@ -596,7 +596,7 @@ class ExprNode(Node):
             self.allocate_temp_result(code)
 
         self.generate_result_code(code)
-        if self.is_temp:
+        if self.is_temp and not (self.type.is_string or self.type.is_pyunicode_ptr):
             # If we are temp we do not need to wait until this node is disposed
             # before disposing children.
             self.generate_subexpr_disposal_code(code)
@@ -611,6 +611,10 @@ class ExprNode(Node):
 
     def generate_disposal_code(self, code):
         if self.is_temp:
+            if self.type.is_string or self.type.is_pyunicode_ptr:
+                # postponed from self.generate_evaluation_code()
+                self.generate_subexpr_disposal_code(code)
+                self.free_subexpr_temps(code)
             if self.result():
                 if self.type.is_pyobject:
                     code.put_decref_clear(self.result(), self.ctype())
@@ -629,7 +633,11 @@ class ExprNode(Node):
 
     def generate_post_assignment_code(self, code):
         if self.is_temp:
-            if self.type.is_pyobject:
+            if self.type.is_string or self.type.is_pyunicode_ptr:
+                # postponed from self.generate_evaluation_code()
+                self.generate_subexpr_disposal_code(code)
+                self.free_subexpr_temps(code)
+            elif self.type.is_pyobject:
                 code.putln("%s = 0;" % self.result())
             elif self.type.is_memoryviewslice:
                 code.putln("%s.memview = NULL;" % self.result())
@@ -2796,7 +2804,10 @@ class IndexNode(ExprNode):
             self.compile_time_value_error(e)
 
     def is_ephemeral(self):
-        return self.base.is_ephemeral()
+        # in most cases, indexing will return a safe reference to an object in a container,
+        # so we consider the result safe if the base object is
+        return self.base.is_ephemeral() or self.base.type in (
+            basestring_type, str_type, bytes_type, unicode_type)
 
     def is_simple(self):
         if self.is_buffer_access or self.memslice_index:
@@ -8455,6 +8466,10 @@ class TypecastNode(ExprNode):
         # either temp or a C cast => no side effects other than the operand's
         return self.operand.is_simple()
 
+    def is_ephemeral(self):
+        # either temp or a C cast => no side effects other than the operand's
+        return self.operand.is_ephemeral()
+
     def nonlocally_immutable(self):
         return self.is_temp or self.operand.nonlocally_immutable()
 
@@ -9014,6 +9029,10 @@ class BinopNode(ExprNode):
     def check_const(self):
         return self.operand1.check_const() and self.operand2.check_const()
 
+    def is_ephemeral(self):
+        return (super(BinopNode, self).is_ephemeral() or
+                self.operand1.is_ephemeral() or self.operand2.is_ephemeral())
+
     def generate_result_code(self, code):
         #print "BinopNode.generate_result_code:", self.operand1, self.operand2 ###
         if self.operand1.type.is_pyobject:
@@ -9648,18 +9667,24 @@ class BoolBinopNode(ExprNode):
                 or self.operand2.compile_time_value(denv)
 
     def coerce_to_boolean(self, env):
-        return BoolBinopNode(
-            self.pos,
-            operator = self.operator,
-            operand1 = self.operand1.coerce_to_boolean(env),
-            operand2 = self.operand2.coerce_to_boolean(env),
-            type = PyrexTypes.c_bint_type,
-            is_temp = self.is_temp)
+        return BoolBinopNode.from_node(
+            self,
+            operator=self.operator,
+            operand1=self.operand1.coerce_to_boolean(env),
+            operand2=self.operand2.coerce_to_boolean(env),
+            type=PyrexTypes.c_bint_type,
+            is_temp=self.is_temp)
 
     def analyse_types(self, env):
         self.operand1 = self.operand1.analyse_types(env)
         self.operand2 = self.operand2.analyse_types(env)
-        self.type = PyrexTypes.independent_spanning_type(self.operand1.type, self.operand2.type)
+        self.type = PyrexTypes.independent_spanning_type(
+            self.operand1.type, self.operand2.type)
+        # note: self.type might be ErrorType, but we allow this here
+        # in order to support eventual coercion to boolean
+        if not self.type.is_pyobject and not self.type.is_error:
+            if self.operand1.is_ephemeral() or self.operand2.is_ephemeral():
+                error(self.pos, "Unsafe C derivative of temporary Python reference used in and/or expression")
         self.operand1 = self.operand1.coerce_to(self.type, env)
         self.operand2 = self.operand2.coerce_to(self.type, env)
 
@@ -9676,6 +9701,9 @@ class BoolBinopNode(ExprNode):
         return self.operand1.check_const() and self.operand2.check_const()
 
     def generate_evaluation_code(self, code):
+        if self.type is error_type:
+            # quite clearly, we did *not* coerce to boolean, but both operand types mismatch
+            error(self.pos, "incompatible types in short-circuiting boolean expression not resolved")
         code.mark_pos(self.pos)
         self.operand1.generate_evaluation_code(code)
         test_result, uses_temp = self.generate_operand1_test(code)
@@ -9703,6 +9731,12 @@ class BoolBinopNode(ExprNode):
         self.operand1.free_temps(code)
         code.putln("}")
 
+    def generate_subexpr_disposal_code(self, code):
+        pass  # nothing to do here, all done in generate_evaluation_code()
+
+    def free_subexpr_temps(self, code):
+        pass  # nothing to do here, all done in generate_evaluation_code()
+
     def generate_operand1_test(self, code):
         #  Generate code to test the truth of the first operand.
         if self.type.is_pyobject:
@@ -9744,6 +9778,9 @@ class CondExprNode(ExprNode):
         else:
             self.constant_result = self.false_val.constant_result
 
+    def is_ephemeral(self):
+        return self.true_val.is_ephemeral() or self.false_val.is_ephemeral()
+
     def analyse_types(self, env):
         self.test = self.test.analyse_types(env).coerce_to_boolean(env)
         self.true_val = self.true_val.analyse_types(env)
@@ -9756,6 +9793,8 @@ class CondExprNode(ExprNode):
             self.true_val.type, self.false_val.type)
         if self.type.is_pyobject:
             self.result_ctype = py_object_type
+        elif self.true_val.is_ephemeral() or self.false_val.is_ephemeral():
+            error(self.pos, "Unsafe C derivative of temporary Python reference used in conditional expression")
         if self.true_val.type.is_pyobject or self.false_val.type.is_pyobject:
             self.true_val = self.true_val.coerce_to(self.type, env)
             self.false_val = self.false_val.coerce_to(self.type, env)
@@ -9787,7 +9826,7 @@ class CondExprNode(ExprNode):
         code.mark_pos(self.pos)
         self.allocate_temp_result(code)
         self.test.generate_evaluation_code(code)
-        code.putln("if (%s) {" % self.test.result() )
+        code.putln("if (%s) {" % self.test.result())
         self.eval_and_get(code, self.true_val)
         code.putln("} else {")
         self.eval_and_get(code, self.false_val)
@@ -9802,6 +9841,13 @@ class CondExprNode(ExprNode):
         expr.generate_post_assignment_code(code)
         expr.free_temps(code)
 
+    def generate_subexpr_disposal_code(self, code):
+        pass  # done explicitly above (cleanup must separately happen within the if/else blocks)
+
+    def free_subexpr_temps(self, code):
+        pass  # done explicitly above (cleanup must separately happen within the if/else blocks)
+
+
 richcmp_constants = {
     "<" : "Py_LT",
     "<=": "Py_LE",
@@ -10871,10 +10917,7 @@ class CoerceFromPyTypeNode(CoercionNode):
             error(arg.pos,
                   "Cannot convert Python object to '%s'" % result_type)
         if self.type.is_string or self.type.is_pyunicode_ptr:
-            if self.arg.is_ephemeral():
-                error(arg.pos,
-                      "Obtaining '%s' from temporary Python value" % result_type)
-            elif self.arg.is_name and self.arg.entry and self.arg.entry.is_pyglobal:
+            if self.arg.is_name and self.arg.entry and self.arg.entry.is_pyglobal:
                 warning(arg.pos,
                         "Obtaining '%s' from externally modifiable global Python value" % result_type,
                         level=1)
@@ -10883,6 +10926,9 @@ class CoerceFromPyTypeNode(CoercionNode):
         # The arg is always already analysed
         return self
 
+    def is_ephemeral(self):
+        return self.type.is_ptr and self.arg.is_ephemeral()
+
     def generate_result_code(self, code):
         function = self.type.from_py_function
         operand = self.arg.py_result()
@@ -11105,6 +11151,11 @@ class CloneNode(CoercionNode):
             self.entry = self.arg.entry
         return self
 
+    def coerce_to(self, dest_type, env):
+        if self.arg.is_literal:
+            return self.arg.coerce_to(dest_type, env)
+        return super(CloneNode, self).coerce_to(dest_type, env)
+
     def is_simple(self):
         return True # result is always in a temp (or a name)
 

Cython/Compiler/ModuleNode.py

@@ -2112,6 +2112,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         self.body.generate_execution_code(code)
 
+        code.putln()
+        code.putln("/*--- Wrapped vars code ---*/")
+        self.generate_wrapped_entries_code(env, code)
+        code.putln()
+
         if Options.generate_cleanup_code:
             code.globalstate.use_utility_code(
                 UtilityCode.load_cached("RegisterModuleCleanup", "ModuleSetupCode.c"))
@@ -2370,6 +2375,25 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 if entry.used:
                     entry.type.global_init_code(entry, code)
 
+    def generate_wrapped_entries_code(self, env, code):
+        for name, entry in env.entries.items():
+            if (entry.create_wrapper
+                    and not entry.is_type
+                    and entry.scope is env):
+                if not entry.type.create_to_py_utility_code(env):
+                    error(entry.pos, "Cannot convert '%s' to Python object" % entry.type)
+                code.putln("{")
+                code.putln("PyObject* wrapped = %s(%s);"  % (
+                    entry.type.to_py_function,
+                    entry.cname))
+                code.putln(code.error_goto_if_null("wrapped", entry.pos))
+                code.putln(
+                    'if (__Pyx_SetAttrString(%s, "%s", wrapped) < 0) %s;' % (
+                        env.module_cname,
+                        name,
+                        code.error_goto(entry.pos)))
+                code.putln("}")
+
     def generate_c_variable_export_code(self, env, code):
         # Generate code to create PyCFunction wrappers for exported C functions.
         entries = []

Cython/Compiler/Nodes.py

@@ -1424,9 +1424,10 @@ class CEnumDefNode(StatNode):
     #  cname          string or None
     #  items          [CEnumDefItemNode]
     #  typedef_flag   boolean
-    #  visibility     "public" or "private"
+    #  visibility     "public" or "private" or "extern"
     #  api            boolean
     #  in_pxd         boolean
+    #  create_wrapper boolean
     #  entry          Entry
 
     child_attrs = ["items"]
@@ -1434,7 +1435,8 @@ class CEnumDefNode(StatNode):
     def declare(self, env):
          self.entry = env.declare_enum(self.name, self.pos,
              cname = self.cname, typedef_flag = self.typedef_flag,
-             visibility = self.visibility, api = self.api)
+             visibility = self.visibility, api = self.api,
+             create_wrapper = self.create_wrapper)
 
     def analyse_declarations(self, env):
         if self.items is not None:
@@ -1479,7 +1481,8 @@ class CEnumDefItemNode(StatNode):
                 self.value = self.value.analyse_const_expression(env)
         entry = env.declare_const(self.name, enum_entry.type,
             self.value, self.pos, cname = self.cname,
-            visibility = enum_entry.visibility, api = enum_entry.api)
+            visibility = enum_entry.visibility, api = enum_entry.api,
+            create_wrapper = enum_entry.create_wrapper)
         enum_entry.enum_values.append(entry)
 
 
@@ -4590,7 +4593,11 @@ class AssignmentNode(StatNode):
     #  to any of the left hand sides.
 
     def analyse_expressions(self, env):
-        return self.analyse_types(env)
+        node = self.analyse_types(env)
+        if isinstance(node, AssignmentNode):
+            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
+                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
+        return node
 
 #       def analyse_expressions(self, env):
 #           self.analyse_expressions_1(env)
@@ -4751,41 +4758,56 @@ class CascadedAssignmentNode(AssignmentNode):
     #
     #  coerced_rhs_list   [ExprNode]   RHS coerced to type of each LHS
 
-    child_attrs = ["lhs_list", "rhs", "coerced_rhs_list"]
+    child_attrs = ["lhs_list", "rhs", "coerced_values", "coerced_rhs_list"]
     coerced_rhs_list = None
+    coerced_values = None
 
     def analyse_declarations(self, env):
         for lhs in self.lhs_list:
             lhs.analyse_target_declaration(env)
 
-    def analyse_types(self, env, use_temp = 0):
+    def analyse_types(self, env, use_temp=0):
         from .ExprNodes import CloneNode, ProxyNode
 
+        lhs_types = set()
+        for lhs in self.lhs_list:
+            lhs.analyse_target_types(env)
+            lhs.gil_assignment_check(env)
+            lhs_types.add(lhs.type)
+
         rhs = self.rhs.analyse_types(env)
-        if use_temp or rhs.is_attribute or (
-                not rhs.is_name and not rhs.is_literal and
-                rhs.type.is_pyobject):
+        if len(lhs_types) == 1:
+            # common special case: only one type needed on the LHS => coerce only once
+            rhs = rhs.coerce_to(lhs_types.pop(), env)
+
+        if not rhs.is_name and not rhs.is_literal and (
+                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
             rhs = rhs.coerce_to_temp(env)
         else:
             rhs = rhs.coerce_to_simple(env)
-        self.rhs = ProxyNode(rhs)
+        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs
+
+        self.coerced_values = []
+        coerced_values = {}
+        for lhs in self.lhs_list:
+            if lhs.type not in coerced_values and lhs.type != rhs.type:
+                rhs = CloneNode(self.rhs).coerce_to(lhs.type, env)
+                self.coerced_values.append(rhs)
+                coerced_values[lhs.type] = rhs
 
         self.coerced_rhs_list = []
         for lhs in self.lhs_list:
-            lhs.analyse_target_types(env)
-            lhs.gil_assignment_check(env)
-            rhs = CloneNode(self.rhs)
-            rhs = rhs.coerce_to(lhs.type, env)
-            self.coerced_rhs_list.append(rhs)
+            rhs = coerced_values.get(lhs.type, self.rhs)
+            self.coerced_rhs_list.append(CloneNode(rhs))
         return self
 
     def generate_rhs_evaluation_code(self, code):
         self.rhs.generate_evaluation_code(code)
 
     def generate_assignment_code(self, code):
-        for i in range(len(self.lhs_list)):
-            lhs = self.lhs_list[i]
-            rhs = self.coerced_rhs_list[i]
+        for rhs in self.coerced_values:
+            rhs.generate_evaluation_code(code)
+        for lhs, rhs in zip(self.lhs_list, self.coerced_rhs_list):
             rhs.generate_evaluation_code(code)
             lhs.generate_assignment_code(rhs, code)
             # Assignment has disposed of the cloned RHS
@@ -4796,9 +4818,11 @@ class CascadedAssignmentNode(AssignmentNode):
         self.rhs.generate_function_definitions(env, code)
 
     def annotate(self, code):
-        for i in range(len(self.lhs_list)):
-            self.lhs_list[i].annotate(code)
-            self.coerced_rhs_list[i].annotate(code)
+        for rhs in self.coerced_values:
+            rhs.annotate(code)
+        for lhs, rhs in zip(self.lhs_list, self.coerced_rhs_list):
+            lhs.annotate(code)
+            rhs.annotate(code)
         self.rhs.annotate(code)
 
 

Cython/Compiler/Optimize.py

@@ -1327,6 +1327,24 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform):
             return pos_args[0]
         return node
 
+    def _handle_simple_function_slice(self, node, pos_args):
+        arg_count = len(pos_args)
+        start = step = None
+        if arg_count == 1:
+            stop, = pos_args
+        elif arg_count == 2:
+            start, stop = pos_args
+        elif arg_count == 3:
+            start, stop, step = pos_args
+        else:
+            self._error_wrong_arg_count('slice', node, pos_args)
+            return node
+        return ExprNodes.SliceNode(
+            node.pos,
+            start=start or ExprNodes.NoneNode(node.pos),
+            stop=stop,
+            step=step or ExprNodes.NoneNode(node.pos))
+
     class YieldNodeCollector(Visitor.TreeVisitor):
         def __init__(self):
             Visitor.TreeVisitor.__init__(self)

Cython/Compiler/Parsing.py

@@ -2622,7 +2622,8 @@ def p_cdef_statement(s, ctx):
         if ctx.level not in ('module', 'module_pxd'):
             error(pos, "C struct/union/enum definition not allowed here")
         if ctx.overridable:
-            error(pos, "C struct/union/enum cannot be declared cpdef")
+            if s.systring != 'enum':
+                error(pos, "C struct/union cannot be declared cpdef")
         return p_struct_enum(s, pos, ctx)
     elif s.sy == 'IDENT' and s.systring == 'fused':
         return p_fused_definition(s, pos, ctx)
@@ -2679,6 +2680,7 @@ def p_c_enum_definition(s, pos, ctx):
     return Nodes.CEnumDefNode(
         pos, name = name, cname = cname, items = items,
         typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
+        create_wrapper = ctx.overridable,
         api = ctx.api, in_pxd = ctx.level == 'module_pxd')
 
 def p_c_enum_line(s, ctx, items):

Cython/Compiler/PyrexTypes.py

@@ -3715,6 +3715,9 @@ def independent_spanning_type(type1, type2):
         return py_object_type
     span_type = _spanning_type(type1, type2)
     if span_type is None:
+        if type1.is_ptr and type2.is_ptr:
+            # incompatible pointers, void* will do as a result
+            return c_void_ptr_type
         return error_type
     return span_type
 

Cython/Compiler/Symtab.py

@@ -407,7 +407,7 @@ class Scope(object):
         """ Return the module-level scope containing this scope. """
         return self.outer_scope.builtin_scope()
 
-    def declare(self, name, cname, type, pos, visibility, shadow = 0, is_type = 0):
+    def declare(self, name, cname, type, pos, visibility, shadow = 0, is_type = 0, create_wrapper = 0):
         # Create new entry, and add to dictionary if
         # name is not None. Reports a warning if already
         # declared.
@@ -424,6 +424,7 @@ class Scope(object):
                 error(pos, "'%s' redeclared " % name)
         entry = Entry(name, cname, type, pos = pos)
         entry.in_cinclude = self.in_cinclude
+        entry.create_wrapper = create_wrapper
         if name:
             entry.qualified_name = self.qualify_name(name)
 #            if name in entries and self.is_cpp():
@@ -444,14 +445,14 @@ class Scope(object):
     def qualify_name(self, name):
         return EncodedString("%s.%s" % (self.qualified_name, name))
 
-    def declare_const(self, name, type, value, pos, cname = None, visibility = 'private', api = 0):
+    def declare_const(self, name, type, value, pos, cname = None, visibility = 'private', api = 0, create_wrapper = 0):
         # Add an entry for a named constant.
         if not cname:
             if self.in_cinclude or (visibility == 'public' or api):
                 cname = name
             else:
                 cname = self.mangle(Naming.enum_prefix, name)
-        entry = self.declare(name, cname, type, pos, visibility)
+        entry = self.declare(name, cname, type, pos, visibility, create_wrapper = create_wrapper)
         entry.is_const = 1
         entry.value_node = value
         return entry
@@ -588,7 +589,7 @@ class Scope(object):
                 entry.name, entry.visibility))
 
     def declare_enum(self, name, pos, cname, typedef_flag,
-            visibility = 'private', api = 0):
+            visibility = 'private', api = 0, create_wrapper = 0):
         if name:
             if not cname:
                 if self.in_cinclude or (visibility == 'public' or api):
@@ -600,6 +601,7 @@ class Scope(object):
             type = PyrexTypes.c_anon_enum_type
         entry = self.declare_type(name, type, pos, cname = cname,
             visibility = visibility, api = api)
+        entry.create_wrapper = create_wrapper
         entry.enum_values = []
         self.sue_entries.append(entry)
         return entry

Cython/Includes/posix/stat.pxd

@@ -4,8 +4,19 @@ from posix.types cimport (blkcnt_t, blksize_t, dev_t, gid_t, ino_t, mode_t,
 
 cdef extern from "sys/stat.h" nogil:
     cdef struct struct_stat "stat":
-        dev_t st_dev
-        ino_t st_ino
+        dev_t   st_dev
+        ino_t   st_ino
+        mode_t  st_mode
+        nlink_t st_nlink
+        uid_t   st_uid
+        gid_t   st_gid
+        dev_t   st_rdev
+        off_t   st_size
+        blksize_t st_blksize
+        blkcnt_t st_blocks
+        time_t  st_atime
+        time_t  st_mtime
+        time_t  st_ctime
 
 # POSIX prescribes including both <sys/stat.h> and <unistd.h> for these
 cdef extern from "unistd.h" nogil:

Tools/cython-mode.el

@@ -46,23 +46,53 @@
   "Keymap used in `cython-mode'.")
 
 (defvar cython-font-lock-keywords
-  `(;; new keywords in Cython language
+  `(;; ctypedef statement: "ctypedef (...type... alias)?"
+    (,(rx
+       ;; keyword itself
+       symbol-start (group "ctypedef")
+       ;; type specifier: at least 1 non-identifier symbol + 1 identifier
+       ;; symbol and anything but a comment-starter after that.
+       (opt (regexp "[^a-zA-z0-9_\n]+[a-zA-Z0-9_][^#\n]*")
+            ;; type alias: an identifier
+            symbol-start (group (regexp "[a-zA-Z_]+[a-zA-Z0-9_]*"))
+            ;; space-or-comments till the end of the line
+            (* space) (opt "#" (* nonl)) line-end))
+     (1 font-lock-keyword-face)
+     (2 font-lock-type-face nil 'noerror))
+    ;; new keywords in Cython language
     (,(rx symbol-start
-          (or "by" "cdef" "cimport" "cpdef" "ctypedef" "enum" "except?"
+          (or "by" "cdef" "cimport" "cpdef"
               "extern" "gil" "include" "nogil" "property" "public"
-              "readonly" "struct" "union" "DEF" "IF" "ELIF" "ELSE")
+              "readonly" "DEF" "IF" "ELIF" "ELSE"
+              "new" "del" "cppclass" "namespace" "const"
+              "__stdcall" "__cdecl" "__fastcall" "inline" "api")
           symbol-end)
-     1 font-lock-keyword-face)
+     . font-lock-keyword-face)
+    ;; Question mark won't match at a symbol-end, so 'except?' must be
+    ;; special-cased.  It's simpler to handle it separately than weaving it
+    ;; into the lengthy list of other keywords.
+    (,(rx symbol-start "except?") . font-lock-keyword-face)
     ;; C and Python types (highlight as builtins)
     (,(rx symbol-start
-          (or "NULL" "bint" "char" "dict" "double" "float" "int" "list"
-              "long" "object" "Py_ssize_t" "short" "size_t" "void")
+          (or
+           "object" "dict" "list"
+           ;; basic c type names
+           "void" "char" "int" "float" "double" "bint"
+           ;; longness/signed/constness
+           "signed" "unsigned" "long" "short"
+           ;; special basic c types
+           "size_t" "Py_ssize_t" "Py_UNICODE" "Py_UCS4" "ssize_t" "ptrdiff_t")
           symbol-end)
-     1 font-lock-builtin-face)
+     . font-lock-builtin-face)
+    (,(rx symbol-start "NULL" symbol-end)
+     . font-lock-constant-face)
     ;; cdef is used for more than functions, so simply highlighting the next
     ;; word is problematic. struct, enum and property work though.
-    ("\\_<\\(?:struct\\|enum\\)[ \t]+\\([a-zA-Z_]+[a-zA-Z0-9_]*\\)"
-     1 py-class-name-face)
+    (,(rx symbol-start
+          (group (or "struct" "enum" "union"
+                     (seq "ctypedef" (+ space "fused"))))
+          (+ space) (group (regexp "[a-zA-Z_]+[a-zA-Z0-9_]*")))
+     (1 font-lock-keyword-face prepend) (2 font-lock-type-face))
     ("\\_<property[ \t]+\\([a-zA-Z_]+[a-zA-Z0-9_]*\\)"
      1 font-lock-function-name-face))
   "Additional font lock keywords for Cython mode.")
@@ -249,8 +279,7 @@ Finds end of innermost nested class or method definition."
   "Major mode for Cython development, derived from Python mode.
 
 \\{cython-mode-map}"
-  (setcar font-lock-defaults
-          (append python-font-lock-keywords cython-font-lock-keywords))
+  (font-lock-add-keywords nil cython-font-lock-keywords)
   (set (make-local-variable 'outline-regexp)
        (rx (* space) (or "class" "def" "cdef" "cpdef" "elif" "else" "except" "finally"
                          "for" "if" "try" "while" "with")

runtests.py

@@ -1929,13 +1929,18 @@ def runtests(options, cmd_args, coverage=None):
         coverage.stop()
         ignored_modules = set(
             'Cython.Compiler.' + name
-            for name in ('Version', 'DebugFlags', 'CmdLine'))
-        modules = [module for name, module in sys.modules.items()
-                   if module is not None and
-                   name.startswith('Cython.') and
-                   '.Tests' not in name and
-                   name not in ignored_modules and
-                   not name.startswith('Cython.Runtime')]
+            for name in ('Version', 'DebugFlags', 'CmdLine')) | set(
+            'Cython.' + name
+            for name in ('Debugging',))
+        ignored_packages = ['Cython.Runtime', 'Cython.Tempita']
+        modules = [
+            module for name, module in sys.modules.items()
+            if module is not None and
+            name.startswith('Cython.') and
+            '.Tests' not in name and
+            name not in ignored_modules and
+            not any(name.startswith(package) for package in ignored_packages)
+        ]
         if options.coverage:
             coverage.report(modules, show_missing=0)
         if options.coverage_xml:

tests/errors/charptr_from_temp.pyx

@@ -18,6 +18,13 @@ cptr = s
 # temp => error
 cptr = s + b"cba"
 
+# indexing => error (but not clear enough to make it a compiler error)
+cptr = s[0]
+cdef char* x = <char*>s[0]
+
+# slicing => error
+cptr = s[:2]
+
 
 cdef unicode  c_u = u"abc"
 u = u"abc"
@@ -37,9 +44,23 @@ cuptr = u
 cuptr = u + u"cba"
 
 
+# coercion in conditional expression => ok
+boolval = list(u)
+cptr = c_s if boolval else c_s
+
+# temp in conditional expression => error
+cptr = s + b'x' if boolval else s + b'y'
+
+
 _ERRORS = """
 16:8: Obtaining 'char *' from externally modifiable global Python value
-19:9: Obtaining 'char *' from temporary Python value
-34:9: Obtaining 'Py_UNICODE *' from externally modifiable global Python value
-37:10: Obtaining 'Py_UNICODE *' from temporary Python value
+19:9: Storing unsafe C derivative of temporary Python reference
+#22:8: Storing unsafe C derivative of temporary Python reference
+#23:5: Storing unsafe C derivative of temporary Python reference
+#23:15: Casting temporary Python object to non-numeric non-Python type
+26:8: Storing unsafe C derivative of temporary Python reference
+41:9: Obtaining 'Py_UNICODE *' from externally modifiable global Python value
+44:10: Storing unsafe C derivative of temporary Python reference
+52:7: Storing unsafe C derivative of temporary Python reference
+52:7: Unsafe C derivative of temporary Python reference used in conditional expression
 """

tests/errors/e_tempcast.pyx

@@ -9,4 +9,5 @@ def foo(obj):
 
 _ERRORS = u"""
 8:5: Casting temporary Python object to non-numeric non-Python type
+8:5: Storing unsafe C derivative of temporary Python reference
 """

tests/memoryview/memslice.pyx

@@ -275,8 +275,6 @@ def cascaded_buffer_assignment(obj):
     >>> A = IntMockBuffer("A", range(6))
     >>> cascaded_buffer_assignment(A)
     acquired A
-    acquired A
-    released A
     released A
     """
     cdef int[:] a, b

tests/run/builtin_slice.pyx

+# mode: run
+
+def slice1(stop):
+    """
+    >>> list(range(8))
+    [0, 1, 2, 3, 4, 5, 6, 7]
+    >>> list(range(10))[slice1(8)]
+    [0, 1, 2, 3, 4, 5, 6, 7]
+    >>> slice1(1)
+    slice(None, 1, None)
+    >>> slice1(10)
+    slice(None, 10, None)
+    >>> slice1(None)
+    slice(None, None, None)
+    >>> slice1(1) == slice(1)
+    True
+    >>> slice1(None) == slice(None)
+    True
+    """
+    return slice(stop)
+
+
+def slice1_const():
+    """
+    >>> slice1_const() == slice(12)
+    True
+    """
+    return slice(12)
+
+
+def slice2(start, stop):
+    """
+    >>> list(range(2, 8))
+    [2, 3, 4, 5, 6, 7]
+    >>> list(range(10))[slice2(2, 8)]
+    [2, 3, 4, 5, 6, 7]
+    >>> slice2(1, 10)
+    slice(1, 10, None)
+    >>> slice2(None, 10)
+    slice(None, 10, None)
+    >>> slice2(4, None)
+    slice(4, None, None)
+    """
+    return slice(start, stop)
+
+
+def slice2_const():
+    """
+    >>> slice2_const() == slice(None, 12)
+    True
+    """
+    return slice(None, 12)
+
+
+def slice3(start, stop, step):
+    """
+    >>> list(range(2, 8, 3))
+    [2, 5]
+    >>> list(range(10))[slice3(2, 8, 3)]
+    [2, 5]
+    >>> slice3(2, None, 3)
+    slice(2, None, 3)
+    >>> slice3(None, 3, 2)
+    slice(None, 3, 2)
+    """
+    return slice(start, stop, step)
+
+
+def slice3_const():
+    """
+    >>> slice3_const() == slice(12, None, 34)
+    True
+    """
+    return slice(12, None, 34)

tests/run/cascaded_typed_assignments_T466.pyx

@@ -24,6 +24,18 @@ def simple_parallel_int_mix():
     ai, bi = al, bl = ao, bo = c = d = [1,2]
     return ao, bo, ai, bi, al, bl, c, d
 
+def simple_parallel_int_mix_recursive():
+    """
+    >>> simple_parallel_int_mix_recursive()
+    (1, 2, 3, 1, [2, 3], 1, 2, 3, 1, 2, 3, [1, [2, 3]], [1, [2, 3]])
+    """
+    cdef int ai, bi, ci
+    cdef long al, bl, cl
+    cdef object ao, bo, co
+    cdef object xo, yo
+    ai, [bi, ci] = al, [bl, cl] = xo, yo = ao, [bo, co] = c = d = [1, [2, 3]]
+    return ao, bo, co, xo, yo, ai, bi, ci, al, bl, cl, c, d
+
 cdef int called = 0
 
 cdef char* get_string():

tests/run/cascadedassignment.pyx

 import cython
 
+@cython.test_fail_if_path_exists(
+    '//CascadedAssignmentNode//CoerceFromPyTypeNode',
+    '//CascadedAssignmentNode//CoerceToPyTypeNode',
+)
+@cython.test_assert_path_exists('//CascadedAssignmentNode')
 def test_cascaded_assignment_simple():
     """
     >>> test_cascaded_assignment_simple()
@@ -8,6 +13,11 @@ def test_cascaded_assignment_simple():
     a = b = c = 5
     return a
 
+@cython.test_fail_if_path_exists(
+    '//CascadedAssignmentNode//CoerceFromPyTypeNode',
+    '//CascadedAssignmentNode//CoerceToPyTypeNode',
+)
+@cython.test_assert_path_exists('//CascadedAssignmentNode')
 def test_cascaded_assignment_typed():
     """
     >>> test_cascaded_assignment_typed()
@@ -46,4 +56,3 @@ def test_cascaded_assignment_evaluate_expr():
     """
     a = b = c = float(expr())
     return a, b, c
-

tests/run/charptr_from_temp.pyx

+# mode: run
+
+from cpython.version cimport PY_MAJOR_VERSION
+
+cdef bint IS_PY2 = PY_MAJOR_VERSION == 2
+
+
+cdef cfunc1(char* s):
+    if IS_PY2:
+        return s
+    else:
+        return s.decode('ASCII')
+
+
+cdef cfunc3(int x, char* s, object y):
+    return cfunc1(s)
+
+
+def test_one_arg_indexing(s):
+    """
+    >>> test_one_arg_indexing(b'xyz')
+    'y'
+    """
+    cfunc1(s[0]) if IS_PY2 else cfunc1(s[:1])
+    z = cfunc1(s[2]) if IS_PY2 else cfunc1(s[2:])
+    assert z == 'z', repr(z)
+    return cfunc1(s[1]) if IS_PY2 else cfunc1(s[1:2])
+
+
+def test_more_args_indexing(s):
+    """
+    >>> test_more_args_indexing(b'xyz')
+    'y'
+    """
+    cfunc3(1, s[0 if IS_PY2 else slice(0,1)], 6.5)
+    z = cfunc3(2, s[2 if IS_PY2 else slice(2,None)], 'abc' * 2)
+    assert z == 'z', repr(z)
+    return cfunc3(3, s[1 if IS_PY2 else slice(1,2)], 1)
+
+
+def test_one_arg_slicing(s):
+    """
+    >>> test_one_arg_slicing(b'xyz')
+    'y'
+    """
+    cfunc1(s[:2])
+    z = cfunc1(s[2:])
+    assert z == 'z', repr(z)
+    return cfunc1(s[1:2])
+
+
+def test_more_args_slicing(s):
+    """
+    >>> test_more_args_slicing(b'xyz')
+    'y'
+    """
+    cfunc3(1, s[:2], 'abc')
+    z = cfunc3(123, s[2:], 5)
+    assert z == 'z', repr(z)
+    return cfunc3(2, s[1:2], 1.4)
+
+
+def test_one_arg_adding(s):
+    """
+    >>> test_one_arg_adding(b'xyz')
+    'abxyzqr'
+    """
+    return cfunc1(b"a" + b"b" + s + b"q" + b"r")
+
+
+def test_more_args_adding(s):
+    """
+    >>> test_more_args_adding(b'xyz')
+    'abxyzqr'
+    """
+    return cfunc3(1, b"a" + b"b" + s + b"q" + b"r", 'xyz%d' % 3)
+
+
+cdef char* ret_charptr(char* s):
+    return s
+
+
+def test_charptr_and_charptr_func(char* s):
+    """
+    >>> test_charptr_and_charptr_func(b'abc') == b'abc'
+    True
+    """
+    return s and ret_charptr(s)
+
+
+def test_charptr_and_ucharptr(char* s):
+    """
+    >>> test_charptr_and_ucharptr(b'abc') == b'abc'
+    True
+    """
+    return s and <unsigned char*>s

tests/run/cpdef_enums.pxd

+cdef extern from *:
+    cpdef enum ExternPxdEnum:
+        FOUR "4"
+        EIGHT "8"
+
+    cdef enum ExternSecretPxdEnum:
+        SIXTEEN "16"
+
+cpdef enum PxdEnum:
+    RANK_0 = 11
+    RANK_1 = 37
+    RANK_2 = 389
+
+cdef enum PxdSecretEnum:
+    RANK_3 = 5077

tests/run/cpdef_enums.pyx

+"""
+>>> ONE, TEN, HUNDRED
+(1, 10, 100)
+>>> THOUSAND        # doctest: +ELLIPSIS
+Traceback (most recent call last):
+NameError: ...name 'THOUSAND' is not defined
+
+>>> TWO, THREE, FIVE
+(2, 3, 5)
+>>> SEVEN           # doctest: +ELLIPSIS
+Traceback (most recent call last):
+NameError: ...name 'SEVEN' is not defined
+
+>>> FOUR, EIGHT
+(4, 8)
+>>> SIXTEEN        # doctest: +ELLIPSIS
+Traceback (most recent call last):
+NameError: ...name 'SIXTEEN' is not defined
+
+>>> RANK_0, RANK_1, RANK_2
+(11, 37, 389)
+>>> RANK_3         # doctest: +ELLIPSIS
+Traceback (most recent call last):
+NameError: ...name 'RANK_3' is not defined
+"""
+
+
+cdef extern from *:
+    cpdef enum ExternPyxEnum:
+        ONE "1"
+        TEN "10"
+        HUNDRED "100"
+
+    cdef enum ExternSecretPyxEnum:
+        THOUSAND "1000"
+
+cpdef enum PyxEnum:
+    TWO = 2
+    THREE = 3
+    FIVE = 5
+
+cdef enum SecretPyxEnum:
+    SEVEN = 7

tests/run/cpdef_enums_import.srctree

+PYTHON setup.py build_ext --inplace
+PYTHON -c "import import_enums_test"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+
+from distutils.core import setup
+
+setup(
+  ext_modules = cythonize(["enums.pyx", "no_enums.pyx"]),
+)
+
+######## enums.pyx ########
+
+cpdef enum:
+    BAR
+
+cpdef foo(): pass
+
+######## enums.pxd ########
+
+cpdef enum:
+    FOO
+
+cpdef foo()
+
+######## no_enums.pyx ########
+
+from enums cimport *
+
+######## import_enums_test.py ########
+
+# We can import enums with a star import.
+from enums import *
+
+print(dir())
+assert 'BAR' in dir() and 'FOO' in dir()
+
+# enums not generated in the wrong module
+import no_enums
+print(dir(no_enums))
+assert 'FOO' not in dir(no_enums)
+assert 'foo' not in dir(no_enums)
+

tests/run/if_else_expr.pyx

@@ -40,5 +40,5 @@ def test_charptr_coercion(x):
     >>> print(test_charptr_coercion(False))
     def
     """
-    cdef char* s = 'abc' if x else 'def'
+    cdef char* s = b'abc' if x else b'def'
     return s.decode('ascii')

tests/run/pointers.pyx

+cimport cython
 
 cdef char* c_string = b'abcdefg'
 cdef void* void_ptr = c_string
@@ -60,3 +61,29 @@ def bool_binop_truth(int x):
         print True
     if c_string and x or not (void_ptr or int_ptr and float_ptr) or x:
         print True
+
+
+def binop_voidptr(int x, long y, char* z):
+    """
+    >>> binop_voidptr(1, 3, b'abc')
+    'void *'
+    """
+    result = &x and &y and z
+    return cython.typeof(result)
+
+
+def cond_expr_voidptr(int x, long y, char* z):
+    """
+    >>> cond_expr_voidptr(0, -1, b'abc')
+    ('void *', 0)
+    >>> cond_expr_voidptr(-1, 0, b'abc')
+    ('void *', -1)
+    >>> cond_expr_voidptr(-1, 0, b'')
+    ('void *', 0)
+    >>> cond_expr_voidptr(0, -1, b'')
+    ('void *', -1)
+    """
+    result = &x if len(z) else &y
+    assert sizeof(long) >= sizeof(int)
+    assert -1 == <int>(-1L)
+    return cython.typeof(result), (<int*>result)[0]