Merge pull request #1370 from gevent/issue1318

Avoid a memory leak when wrapping an io.BufferedWriter around a socket.

CHANGES.rst

@@ -37,6 +37,8 @@
   Python 2 for namespace packages, so if ``futures`` is installed this
   will likely be the case.
 
+- Python 2: Avoid a memory leak when an `io.BufferedWriter` is wrapped
+  around a socket. Reported by Damien Tournoud in :issue:`1318`.
 
 1.4.0 (2019-01-04)
 ==================

benchmarks/bench_get_memory.py

+"""
+Benchmarking for getting the memoryview of an object.
+
+https://github.com/gevent/gevent/issues/1318
+"""
+from __future__ import print_function
+
+# pylint:disable=unidiomatic-typecheck
+
+try:
+    xrange
+except NameError:
+    xrange = range
+
+try:
+    buffer
+except NameError:
+    buffer = memoryview
+
+import perf
+
+from gevent._greenlet_primitives import get_memory as cy_get_memory
+
+def get_memory_gevent14(data):
+    try:
+        mv = memoryview(data)
+        if mv.shape:
+            return mv
+        # No shape, probably working with a ctypes object,
+        # or something else exotic that supports the buffer interface
+        return mv.tobytes()
+    except TypeError:
+        # fixes "python2.7 array.array doesn't support memoryview used in
+        # gevent.socket.send" issue
+        # (http://code.google.com/p/gevent/issues/detail?id=94)
+        return buffer(data)
+
+def get_memory_is(data):
+    try:
+        mv = memoryview(data) if type(data) is not memoryview else data
+        if mv.shape:
+            return mv
+        # No shape, probably working with a ctypes object,
+        # or something else exotic that supports the buffer interface
+        return mv.tobytes()
+    except TypeError:
+        # fixes "python2.7 array.array doesn't support memoryview used in
+        # gevent.socket.send" issue
+        # (http://code.google.com/p/gevent/issues/detail?id=94)
+        return buffer(data)
+
+def get_memory_inst(data):
+    try:
+        mv = memoryview(data) if not isinstance(data, memoryview) else data
+        if mv.shape:
+            return mv
+        # No shape, probably working with a ctypes object,
+        # or something else exotic that supports the buffer interface
+        return mv.tobytes()
+    except TypeError:
+        # fixes "python2.7 array.array doesn't support memoryview used in
+        # gevent.socket.send" issue
+        # (http://code.google.com/p/gevent/issues/detail?id=94)
+        return buffer(data)
+
+
+N = 100
+
+DATA = {
+    'bytestring': b'abc123',
+    'bytearray': bytearray(b'abc123'),
+    'memoryview': memoryview(b'abc123'),
+}
+
+
+def test(loops, func, arg):
+    t0 = perf.perf_counter()
+    for __ in range(loops):
+        for _ in xrange(N):
+            func(arg)
+    return perf.perf_counter() - t0
+
+
+def main():
+    runner = perf.Runner()
+    for func, name in (
+            (get_memory_gevent14, 'gevent14-py'),
+            (cy_get_memory, 'inst-cy'),
+            (get_memory_inst, 'inst-py'),
+            (get_memory_is, 'is-py'),
+    ):
+        for arg_name, arg in DATA.items():
+            runner.bench_time_func(
+                '%s - %s' % (name, arg_name),
+                test, func, arg,
+                inner_loops=N
+            )
+
+
+if __name__ == '__main__':
+    main()

src/gevent/__greenlet_primitives.pxd

@@ -45,3 +45,8 @@ cdef class SwitchOutGreenletWithLoop(TrackedRawGreenlet):
 
 
 cpdef list get_reachable_greenlets()
+
+cdef type _memoryview
+cdef type _buffer
+
+cpdef get_memory(data)

src/gevent/_greenlet_primitives.py

@@ -26,6 +26,7 @@ locals()['getcurrent'] = __import__('greenlet').getcurrent
 locals()['greenlet_init'] = lambda: None
 locals()['_greenlet_switch'] = greenlet.switch
 
+
 __all__ = [
     'TrackedRawGreenlet',
     'SwitchOutGreenletWithLoop',
@@ -81,6 +82,47 @@ def get_reachable_greenlets():
         if isinstance(x, greenlet) and not getattr(x, 'greenlet_tree_is_ignored', False)
     ]
 
+# Cache the global memoryview so cython can optimize.
+_memoryview = memoryview
+try:
+    if isinstance(__builtins__, dict):
+        # Pure-python mode on CPython
+        _buffer = __builtins__['buffer']
+    else:
+        # Cythonized mode, or PyPy
+        _buffer = __builtins__.buffer
+except (AttributeError, KeyError):
+    # Python 3.
+    _buffer = memoryview
+
+def get_memory(data):
+    # On Python 2, memoryview(memoryview()) can leak in some cases,
+    # notably when an io.BufferedWriter object produced the memoryview.
+    # So we need to check to see if we already have one before we convert.
+    # We do this in Cython to mitigate the performance cost (which turns out to be a
+    # net win.)
+
+    # We don't specifically test for this leak.
+
+    # https://github.com/gevent/gevent/issues/1318
+    try:
+        mv = _memoryview(data) if not isinstance(data, _memoryview) else data
+        if mv.shape:
+            return mv
+        # No shape, probably working with a ctypes object,
+        # or something else exotic that supports the buffer interface
+        return mv.tobytes()
+    except TypeError:
+        # fixes "python2.7 array.array doesn't support memoryview used in
+        # gevent.socket.send" issue
+        # (http://code.google.com/p/gevent/issues/detail?id=94)
+        if _buffer is _memoryview:
+            # Py3
+            raise
+        return _buffer(data)
+
+
+
 def _init():
     greenlet_init() # pylint:disable=undefined-variable
 

src/gevent/_socket2.py

@@ -69,20 +69,7 @@ else:
             super(_fileobject, self).close()
 
 
-def _get_memory(data):
-    try:
-        mv = memoryview(data)
-        if mv.shape:
-            return mv
-        # No shape, probably working with a ctypes object,
-        # or something else exotic that supports the buffer interface
-        return mv.tobytes()
-    except TypeError:
-        # fixes "python2.7 array.array doesn't support memoryview used in
-        # gevent.socket.send" issue
-        # (http://code.google.com/p/gevent/issues/detail?id=94)
-        return buffer(data)
-
+from gevent._greenlet_primitives import get_memory as _get_memory
 
 class _closedsocket(object):
     __slots__ = []

src/gevent/_socket3.py

@@ -40,14 +40,7 @@ __dns__ = _socketcommon.__dns__
 
 SocketIO = __socket__.SocketIO # pylint:disable=no-member
 
-
-def _get_memory(data):
-    mv = memoryview(data)
-    if mv.shape:
-        return mv
-    # No shape, probably working with a ctypes object,
-    # or something else exotic that supports the buffer interface
-    return mv.tobytes()
+from gevent._greenlet_primitives import get_memory as _get_memory
 
 timeout_default = object()
 

src/gevent/tests/test__socket_dns.py

@@ -234,7 +234,10 @@ class TestCase(greentest.TestCase):
 
     def tearDown(self):
         if not self.verbose_dns:
-            del gevent.get_hub().exception_stream
+            try:
+                del gevent.get_hub().exception_stream
+            except AttributeError:
+                pass # Happens under leak tests
         super(TestCase, self).tearDown()
 
     def should_log_results(self, result1, result2):