include fast list/tuple iteration in dict iteration optimisation to prevent...

include fast list/tuple iteration in dict iteration optimisation to prevent performance regressions for the too-optimistic optimisation case

Cython/Utility/Optimize.c

@@ -310,36 +310,41 @@ static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t
 //@requires: ObjectHandling.c::UnpackTuple2
 //@requires: ObjectHandling.c::IterFinish
 
-static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* dict, int is_dict, PyObject* method_name,
+static CYTHON_INLINE PyObject* __Pyx_dict_iterator(PyObject* iterable, int is_dict, PyObject* method_name,
                                                    Py_ssize_t* p_orig_length, int* p_is_dict) {
 #if !CYTHON_COMPILING_IN_PYPY
-    if (is_dict || likely(PyDict_CheckExact(dict))) {
+    if (is_dict || likely(PyDict_CheckExact(iterable))) {
         *p_is_dict = 1;
-        *p_orig_length = PyDict_Size(dict);
-        Py_INCREF(dict);
-        return dict;
+        *p_orig_length = PyDict_Size(iterable);
+        Py_INCREF(iterable);
+        return iterable;
     }
 #endif
     *p_is_dict = 0;
     *p_orig_length = 0;
     if (method_name) {
-        dict = PyObject_CallMethodObjArgs(dict, method_name, NULL);
-        if (!dict)
+        iterable = PyObject_CallMethodObjArgs(iterable, method_name, NULL);
+        if (!iterable)
             return NULL;
+#if !CYTHON_COMPILING_IN_PYPY
+        if (PyTuple_CheckExact(iterable) || PyList_CheckExact(iterable))
+            return iterable;
+#endif
     }
-    return PyObject_GetIter(dict);
+    return PyObject_GetIter(iterable);
 }
 
-static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t orig_length, Py_ssize_t* ppos,
+static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* iter_obj, Py_ssize_t orig_length, Py_ssize_t* ppos,
                                               PyObject** pkey, PyObject** pvalue, PyObject** pitem, int is_dict) {
+    PyObject* next_item;
 #if !CYTHON_COMPILING_IN_PYPY
     if (is_dict) {
         PyObject *key, *value;
-        if (unlikely(orig_length != PyDict_Size(dict_or_iter))) {
+        if (unlikely(orig_length != PyDict_Size(iter_obj))) {
             PyErr_SetString(PyExc_RuntimeError, "dictionary changed size during iteration");
             return -1;
         }
-        if (unlikely(!PyDict_Next(dict_or_iter, ppos, &key, &value))) {
+        if (unlikely(!PyDict_Next(iter_obj, ppos, &key, &value))) {
             return 0;
         }
         if (pitem) {
@@ -363,13 +368,26 @@ static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t
             }
         }
         return 1;
-    }
+    } else if (PyTuple_CheckExact(iter_obj)) {
+        Py_ssize_t pos = *ppos;
+        if (unlikely(pos >= PyTuple_GET_SIZE(iter_obj))) return 0;
+        *ppos = pos + 1;
+        next_item = PyTuple_GET_ITEM(iter_obj, pos);
+        Py_INCREF(next_item);
+    } else if (PyList_CheckExact(iter_obj)) {
+        Py_ssize_t pos = *ppos;
+        if (unlikely(pos >= PyList_GET_SIZE(iter_obj))) return 0;
+        *ppos = pos + 1;
+        next_item = PyList_GET_ITEM(iter_obj, pos);
+        Py_INCREF(next_item);
+    } else
 #endif
     {
-        PyObject* next_item = PyIter_Next(dict_or_iter);
+        next_item = PyIter_Next(iter_obj);
         if (unlikely(!next_item)) {
             return __Pyx_IterFinish();
         }
+    }
     if (pitem) {
         *pitem = next_item;
     } else if (pkey && pvalue) {
@@ -381,7 +399,6 @@ static CYTHON_INLINE int __Pyx_dict_iter_next(PyObject* dict_or_iter, Py_ssize_t
         *pvalue = next_item;
     }
     return 1;
-    }
 }
 
 /////////////// pyobject_as_double.proto ///////////////

tests/run/iterdict.pyx

@@ -43,8 +43,13 @@ def optimistic_iteritems(d):
     >>> optimistic_iteritems({})
     []
     >>> class mydict(object):
-    ...     def iteritems(self): return d.items()
-    >>> optimistic_iteritems(mydict())
+    ...     def __init__(self, t): self.t = t
+    ...     def iteritems(self): return self.t(d.items())
+    >>> optimistic_iteritems(mydict(list))
+    [(10, 0), (11, 1), (12, 2), (13, 3)]
+    >>> optimistic_iteritems(mydict(tuple))
+    [(10, 0), (11, 1), (12, 2), (13, 3)]
+    >>> optimistic_iteritems(mydict(iter))
     [(10, 0), (11, 1), (12, 2), (13, 3)]
     """
     l = []
@@ -103,8 +108,13 @@ def optimistic_iteritems_int(d):
     >>> optimistic_iteritems_int({})
     []
     >>> class mydict(object):
-    ...     def iteritems(self): return d.items()
-    >>> optimistic_iteritems_int(mydict())
+    ...     def __init__(self, t): self.t = t
+    ...     def iteritems(self): return self.t(d.items())
+    >>> optimistic_iteritems_int(mydict(list))
+    [(10, 0), (11, 1), (12, 2), (13, 3)]
+    >>> optimistic_iteritems_int(mydict(tuple))
+    [(10, 0), (11, 1), (12, 2), (13, 3)]
+    >>> optimistic_iteritems_int(mydict(iter))
     [(10, 0), (11, 1), (12, 2), (13, 3)]
 
     >>> optimistic_iteritems_int({'a': 1})
@@ -174,8 +184,17 @@ def optimistic_iterkeys(d):
     >>> optimistic_iterkeys({})
     []
     >>> class mydict(object):
-    ...     def iterkeys(self): return d
-    >>> optimistic_iterkeys(mydict())
+    ...     def __init__(self, t): self.t = t
+    ...     def iterkeys(self): return self.t(d)
+    >>> optimistic_iterkeys(mydict(lambda x:x))
+    [10, 11, 12, 13]
+    >>> optimistic_iterkeys(mydict(lambda x:x.keys()))
+    [10, 11, 12, 13]
+    >>> optimistic_iterkeys(mydict(list))
+    [10, 11, 12, 13]
+    >>> optimistic_iterkeys(mydict(tuple))
+    [10, 11, 12, 13]
+    >>> optimistic_iterkeys(mydict(iter))
     [10, 11, 12, 13]
     """
     l = []
@@ -298,8 +317,15 @@ def optimistic_itervalues(d):
     >>> optimistic_itervalues({})
     []
     >>> class mydict(object):
-    ...     def itervalues(self): return d.values()
-    >>> optimistic_itervalues(mydict())
+    ...     def __init__(self, t): self.t = t
+    ...     def itervalues(self): return self.t(d.values())
+    >>> optimistic_itervalues(mydict(lambda x:x))
+    [0, 1, 2, 3]
+    >>> optimistic_itervalues(mydict(list))
+    [0, 1, 2, 3]
+    >>> optimistic_itervalues(mydict(tuple))
+    [0, 1, 2, 3]
+    >>> optimistic_itervalues(mydict(iter))
     [0, 1, 2, 3]
     """
     l = []