Merge branch 'master' into release

CHANGES.rst

@@ -50,6 +50,9 @@ Features added
 * Modules that cimport many external extension types from other Cython modules
   execute less import requests during module initialisation.
 
+* Constant tuples and slices are deduplicated and only created once per module.
+  (Github issue #2292)
+
 * The coverage plugin considers more C file extensions such as ``.cc`` and ``.cxx``.
   (Github issue #2266)
 

Cython/Compiler/Code.py

@@ -1134,6 +1134,7 @@ class GlobalState(object):
 
         self.const_cnames_used = {}
         self.string_const_index = {}
+        self.dedup_const_index = {}
         self.pyunicode_ptr_const_index = {}
         self.num_const_index = {}
         self.py_constants = []
@@ -1265,13 +1266,19 @@ class GlobalState(object):
             c = self.new_num_const(str_value, 'float', value_code)
         return c
 
-    def get_py_const(self, type, prefix='', cleanup_level=None):
+    def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None):
+        if dedup_key is not None:
+            const = self.dedup_const_index.get(dedup_key)
+            if const is not None:
+                return const
         # create a new Python object constant
         const = self.new_py_const(type, prefix)
         if cleanup_level is not None \
                 and cleanup_level <= Options.generate_cleanup_code:
             cleanup_writer = self.parts['cleanup_globals']
             cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
+        if dedup_key is not None:
+            self.dedup_const_index[dedup_key] = const
         return const
 
     def get_string_const(self, text, py_version=None):
@@ -1792,8 +1799,8 @@ class CCodeWriter(object):
     def get_py_float(self, str_value, value_code):
         return self.globalstate.get_float_const(str_value, value_code).cname
 
-    def get_py_const(self, type, prefix='', cleanup_level=None):
-        return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
+    def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None):
+        return self.globalstate.get_py_const(type, prefix, cleanup_level, dedup_key).cname
 
     def get_string_const(self, text):
         return self.globalstate.get_string_const(text).cname

Cython/Compiler/ExprNodes.py

@@ -187,6 +187,29 @@ def infer_sequence_item_type(env, seq_node, index_node=None, seq_type=None):
             return item_types.pop()
     return None
 
+
+def make_dedup_key(outer_type, item_nodes):
+    """
+    Recursively generate a deduplication key from a sequence of values.
+    Includes Cython node types to work around the fact that (1, 2.0) == (1.0, 2), for example.
+
+    @param outer_type: The type of the outer container.
+    @param item_nodes: A sequence of constant nodes that will be traversed recursively.
+    @return: A tuple that can be used as a dict key for deduplication.
+    """
+    item_keys = [
+        (py_object_type, None) if node is None
+        else make_dedup_key(node.type, node.args) if node.is_sequence_constructor
+        else make_dedup_key(node.type, (node.start, node.stop, node.step)) if node.is_slice
+        else (node.type, node.constant_result) if node.has_constant_result()
+        else None
+        for node in item_nodes
+    ]
+    if None in item_keys:
+        return None
+    return outer_type, tuple(item_keys)
+
+
 # Returns a block of code to translate the exception,
 # plus a boolean indicating whether to check for Python exceptions.
 def get_exception_handler(exception_value):
@@ -5228,7 +5251,8 @@ class SliceNode(ExprNode):
 
     def generate_result_code(self, code):
         if self.is_literal:
-            self.result_code = code.get_py_const(py_object_type, 'slice', cleanup_level=2)
+            dedup_key = make_dedup_key(self.type, (self,))
+            self.result_code = code.get_py_const(py_object_type, 'slice', cleanup_level=2, dedup_key=dedup_key)
             code = code.get_cached_constants_writer()
             code.mark_pos(self.pos)
 
@@ -7961,7 +7985,10 @@ class TupleNode(SequenceNode):
             return
 
         if self.is_literal or self.is_partly_literal:
-            tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2)
+            dedup_key = None
+            if self.is_literal:
+                dedup_key = make_dedup_key(self.type, self.args)
+            tuple_target = code.get_py_const(py_object_type, 'tuple', cleanup_level=2, dedup_key=dedup_key)
             const_code = code.get_cached_constants_writer()
             const_code.mark_pos(self.pos)
             self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal)

Cython/Compiler/Nodes.py

@@ -6337,10 +6337,10 @@ class SwitchCaseNode(StatNode):
     def generate_execution_code(self, code):
         num_conditions = len(self.conditions)
         line_tracing_enabled = code.globalstate.directives['linetrace']
-        for i, cond in enumerate(self.conditions):
+        for i, cond in enumerate(self.conditions, 1):
             code.putln("case %s:" % cond.result())
             code.mark_pos(cond.pos)  # Tracing code must appear *after* the 'case' statement.
-            if line_tracing_enabled and i + 1 < num_conditions:
+            if line_tracing_enabled and i < num_conditions:
                 # Allow fall-through after the line tracing code.
                 code.putln('CYTHON_FALLTHROUGH;')
         self.body.generate_execution_code(code)

tests/run/check_size.srctree

+PYTHON setup.py build_ext --inplace
+PYTHON -c "import runner"
+
+######## setup.py ########
+
+from Cython.Build.Dependencies import cythonize
+from distutils.core import setup
+
+# force the build order
+setup(ext_modules= cythonize("check_size.pyx"))
+
+setup(ext_modules = cythonize("_check_size*.pyx"))
+
+######## check_size_nominal.h ########
+
+#include <Python.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f0;
+    int f1;
+    int f2;
+} FooStructNominal;
+
+#ifdef __cplusplus
+}
+#endif
+
+######## check_size_bigger.h ########
+
+#include <Python.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f0;
+    int f1;
+    int f2;
+    int f3;
+    int f4;
+} FooStructBig;
+
+#ifdef __cplusplus
+}
+#endif
+
+######## check_size_smaller.h ########
+
+#include <Python.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    PyObject_HEAD
+    int f9;
+} FooStructSmall;
+
+#ifdef __cplusplus
+}
+#endif
+
+
+######## check_size.pyx ########
+
+cdef class Foo:
+    cdef public int field0, field1, field2;
+
+    def __init__(self, f0, f1, f2):
+        self.field0 = f0
+        self.field1 = f1
+        self.field2 = f2
+
+######## _check_size0.pyx ########
+
+cdef extern from "check_size_nominal.h":
+
+    ctypedef class check_size.Foo [object FooStructNominal]:
+        cdef:
+            int f0
+            int f1
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f0 + f.f1
+
+######## _check_size1.pyx ########
+
+cdef extern from "check_size_bigger.h":
+
+    ctypedef class check_size.Foo [object FooStructBig]:
+        cdef:
+            int f0
+            int f1
+            int f2
+
+
+cpdef public int testme(Foo f, int f2) except -1:
+    f.f2 = f2
+    return f.f0 + f.f1 + f.f2
+
+######## _check_size2.pyx ########
+
+cdef extern from "check_size_smaller.h":
+
+    ctypedef class check_size.Foo [object FooStructSmall]:
+        cdef:
+            int f9
+
+
+cpdef public int testme(Foo f) except -1:
+    return f.f9
+
+######## runner.py ########
+
+import check_size, _check_size0, warnings
+
+foo = check_size.Foo(23, 123, 1023)
+
+assert foo.field0 == 23
+assert foo.field1 == 123
+
+ret =  _check_size0.testme(foo)
+assert ret == 23 + 123
+
+# ValueError since check_size.Foo's tp_basicsize is smaller than what is needed
+# for FooStructBig. Messing with f2 will access memory outside the struct!
+try:
+    import _check_size1
+    assert False
+except ValueError as e:
+    assert str(e).startswith('check_size.Foo has the wrong size, try recompiling')
+
+# Warining since check_size.Foo's tp_basicsize is larger than what is needed
+# for FooStructSmall. There is "spare", accessing FooStructSmall's fields will
+# never access invalid memory. This can happen, for instance, when using old
+# headers with a newer runtime, or when using an old _check_size2 with a newer
+# check_size, where the developers of check_size are careful to be backward
+# compatible.
+
+with warnings.catch_warnings(record=True) as w:
+    import _check_size2
+    assert len(w) == 1, 'expected one warning, got %d' % len(w)
+    assert str(w[-1].message).startswith('check_size.Foo size changed')
+
+ret = _check_size2.testme(foo)
+assert ret == 23