Reduce memory allocations in C++ to PyList Conversions (GH-4081)

Cython/Utility/CppConvert.pyx

@@ -52,14 +52,32 @@ cdef vector[X] {{cname}}(object o) except *:
 #################### vector.to_py ####################
 
 cdef extern from *:
-    cdef cppclass vector "const std::vector" [T]:
+    cdef cppclass vector "std::vector" [T]:
         size_t size()
         T& operator[](size_t)
 
+cdef extern from "Python.h":
+    void Py_INCREF(object)
+    list PyList_New(Py_ssize_t size)
+    void PyList_SET_ITEM(object list, Py_ssize_t i, object o)
+    cdef Py_ssize_t PY_SSIZE_T_MAX
+
 @cname("{{cname}}")
-cdef object {{cname}}(vector[X]& v):
-    return [v[i] for i in range(v.size())]
+cdef object {{cname}}(const vector[X]& v):
+    if v.size() > <size_t> PY_SSIZE_T_MAX:
+        raise MemoryError()
+
+    o = PyList_New(<Py_ssize_t> v.size())
+
+    cdef Py_ssize_t i
+    cdef object item
 
+    for i in range(v.size()):
+        item = v[i]
+        Py_INCREF(item)
+        PyList_SET_ITEM(o, i, item)
+
+    return o
 
 #################### list.from_py ####################
 
@@ -87,14 +105,32 @@ cdef extern from *:
             bint operator!=(const_iterator)
         const_iterator begin()
         const_iterator end()
+        size_t size()
+
+cdef extern from "Python.h":
+    void Py_INCREF(object)
+    list PyList_New(Py_ssize_t size)
+    void PyList_SET_ITEM(object list, Py_ssize_t i, object o)
+    cdef Py_ssize_t PY_SSIZE_T_MAX
 
 @cname("{{cname}}")
 cdef object {{cname}}(const cpp_list[X]& v):
-    o = []
+    if v.size() > <size_t> PY_SSIZE_T_MAX:
+        raise MemoryError()
+
+    o = PyList_New(<Py_ssize_t> v.size())
+
+    cdef object item
+    cdef Py_ssize_t i = 0
     cdef cpp_list[X].const_iterator iter = v.begin()
+
     while iter != v.end():
-        o.append(cython.operator.dereference(iter))
+        item = cython.operator.dereference(iter)
+        Py_INCREF(item)
+        PyList_SET_ITEM(o, i, item)
         cython.operator.preincrement(iter)
+        i += 1
+
     return o
 
 
@@ -127,12 +163,7 @@ cdef extern from *:
 
 @cname("{{cname}}")
 cdef object {{cname}}(const cpp_set[X]& s):
-    o = set()
-    cdef cpp_set[X].const_iterator iter = s.begin()
-    while iter != s.end():
-        o.add(cython.operator.dereference(iter))
-        cython.operator.preincrement(iter)
-    return o
+    return {v for v in s}
 
 #################### pair.from_py ####################