Merge pull request #1117 from gevent/faster-locals

Speed up gevent.local.local more, especially for subclasses (2-3x faster)

CHANGES.rst

@@ -103,6 +103,12 @@
   Hashemi and Kurt Rose. See :issue:`755`. As always, feedback is
   appreciated.
 
+- Simple subclasses of ``gevent.local.local`` now have the same
+  (substantially improved) performance characteristics of plain
+  ``gevent.local.local`` itself, making them 2 to 3 times faster than
+  before. See :pr:`1117`. If there are any compatibility
+  problems, please open issues.
+
 
 1.3a1 (2018-01-27)
 ==================

setup.py

@@ -55,11 +55,6 @@ SEMAPHORE = Extension(name="gevent._semaphore",
                       depends=['src/gevent/_semaphore.pxd'])
 SEMAPHORE = cythonize1(SEMAPHORE)
 
-LOCAL = Extension(name="gevent.local",
-                  sources=["src/gevent/local.py"],
-                  depends=['src/gevent/local.pxd'])
-LOCAL = cythonize1(LOCAL)
-
 # The sysconfig dir is not enough if we're in a virtualenv
 # See https://github.com/pypa/pip/issues/4610
 include_dirs = [sysconfig.get_path("include")]
@@ -70,6 +65,13 @@ if os.path.exists(venv_include_dir):
     include_dirs.append(venv_include_dir)
 
 
+LOCAL = Extension(name="gevent.local",
+                  sources=["src/gevent/local.py"],
+                  depends=['src/gevent/local.pxd'],
+                  include_dirs=include_dirs)
+LOCAL = cythonize1(LOCAL)
+
+
 GREENLET = Extension(name="gevent.greenlet",
                      sources=[
                          "src/gevent/greenlet.py",

src/gevent/_ident.py

 # -*- coding: utf-8 -*-
 # Copyright 2018 gevent contributors. See LICENSE for details.
+# cython: auto_pickle=False,embedsignature=True,always_allow_keywords=False
 
 from __future__ import absolute_import
 from __future__ import division

src/gevent/greenlet.pxd

@@ -2,12 +2,31 @@
 
 cimport cython
 from gevent._ident cimport IdentRegistry
+cdef bint _greenlet_imported
+cdef bint _PYPY
 
 cdef extern from "greenlet/greenlet.h":
 
     ctypedef class greenlet.greenlet [object PyGreenlet]:
         pass
 
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+cdef void _init()
+
 cdef class SpawnedLink:
     cdef public object callback
 
@@ -87,7 +106,6 @@ cdef class Greenlet(greenlet):
 cdef _greenlet__init__
 cdef get_hub
 cdef wref
-cdef getcurrent
 
 cdef Timeout
 cdef dump_traceback

src/gevent/greenlet.py

@@ -7,9 +7,7 @@ from weakref import ref as wref
 
 
 from greenlet import greenlet
-from greenlet import getcurrent
 
-from gevent._compat import PYPY
 from gevent._compat import reraise
 from gevent._tblib import dump_traceback
 from gevent._tblib import load_traceback
@@ -21,6 +19,9 @@ from gevent.hub import iwait
 from gevent.hub import wait
 from gevent.timeout import Timeout
 
+_PYPY = hasattr(sys, 'pypy_version_info')
+
+
 __all__ = [
     'Greenlet',
     'joinall',
@@ -28,7 +29,15 @@ __all__ = [
 ]
 
 
-if PYPY:
+
+# In Cython, we define these as 'cdef inline' functions. The
+# compilation unit cannot have a direct assignment to them (import
+# is assignment) without generating a 'lvalue is not valid target'
+# error.
+locals()['getcurrent'] = __import__('greenlet').getcurrent
+locals()['greenlet_init'] = lambda: None
+
+if _PYPY:
     import _continuation # pylint:disable=import-error
     _continulet = _continuation.continulet
 
@@ -254,7 +263,7 @@ class Greenlet(greenlet):
         # Failed with exception: (t, v, dump_traceback(tb)))
         self._exc_info = None
 
-        spawner = getcurrent()
+        spawner = getcurrent() # pylint:disable=undefined-variable
         self.spawning_greenlet = wref(spawner)
         try:
             self.spawn_tree_locals = spawner.spawn_tree_locals
@@ -317,7 +326,7 @@ class Greenlet(greenlet):
 
     ### Lifecycle
 
-    if PYPY:
+    if _PYPY:
         # oops - pypy's .dead relies on __nonzero__ which we overriden above
         @property
         def dead(self):
@@ -618,7 +627,7 @@ class Greenlet(greenlet):
         if not block:
             raise Timeout()
 
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
             t = Timeout._start_new_or_dummy(timeout)
@@ -652,7 +661,7 @@ class Greenlet(greenlet):
         if self.ready():
             return
 
-        switch = getcurrent().switch
+        switch = getcurrent().switch # pylint:disable=undefined-variable
         self.rawlink(switch)
         try:
             t = Timeout._start_new_or_dummy(timeout)
@@ -912,3 +921,8 @@ def killall(greenlets, exception=GreenletExit, block=True, timeout=None):
             t.cancel()
     else:
         loop.run_callback(_killall, greenlets, exception)
+
+def _init():
+    greenlet_init() # pylint:disable=undefined-variable
+
+_init()

src/gevent/local.pxd

@@ -2,6 +2,37 @@
 
 cimport cython
 
+cdef bint _PYPY
+cdef ref
+cdef copy
+
+cdef object _marker
+cdef bint _greenlet_imported
+
+
+cdef extern from "greenlet/greenlet.h":
+
+    ctypedef class greenlet.greenlet [object PyGreenlet]:
+        pass
+
+    # These are actually macros and so much be included
+    # (defined) in each .pxd, as are the two functions
+    # that call them.
+    greenlet PyGreenlet_GetCurrent()
+    void PyGreenlet_Import()
+
+cdef inline greenlet getcurrent():
+    return PyGreenlet_GetCurrent()
+
+cdef inline void greenlet_init():
+    global _greenlet_imported
+    if not _greenlet_imported:
+        PyGreenlet_Import()
+        _greenlet_imported = True
+
+
+cdef void _init()
+
 @cython.final
 @cython.internal
 cdef class _wrefdict(dict):
@@ -9,7 +40,7 @@ cdef class _wrefdict(dict):
 
 @cython.final
 @cython.internal
-cdef class _thread_deleted:
+cdef class _greenlet_deleted:
     cdef object idt
     cdef object wrdicts
 
@@ -19,7 +50,7 @@ cdef class _thread_deleted:
 cdef class _local_deleted:
     cdef str key
     cdef object wrthread
-    cdef _thread_deleted thread_deleted
+    cdef _greenlet_deleted greenlet_deleted
 
 @cython.final
 @cython.internal
@@ -30,13 +61,36 @@ cdef class _localimpl:
     cdef object __weakref__
 
 
+@cython.final
+@cython.internal
+cdef class _localimpl_dict_entry:
+    cdef object wrgreenlet
+    cdef dict localdict
+
+@cython.locals(localdict=dict, key=str, greenlet=greenlet,
+               greenlet_deleted=_greenlet_deleted,
+               local_deleted=_local_deleted)
 cdef dict _localimpl_create_dict(_localimpl self)
+
+@cython.locals(entry=_localimpl_dict_entry, greenlet=greenlet)
 cdef inline dict _localimpl_get_dict(_localimpl self)
 
+cdef set _local_attrs
 
 cdef class local:
     cdef _localimpl _local__impl
+    cdef set _local_type_get_descriptors
+    cdef set _local_type_set_or_del_descriptors
+    cdef set _local_type_del_descriptors
+    cdef set _local_type_set_descriptors
+    cdef set _local_type_vars
+    cdef type _local_type
 
+
+@cython.locals(impl=_localimpl,dct=dict)
 cdef inline dict _local_get_dict(local self)
 
+@cython.locals(entry=_localimpl_dict_entry)
 cdef _local__copy_dict_from(local self, _localimpl impl, dict duplicate)
+
+cdef tuple _local_find_descriptors(local self)

src/greentest/test__local.py

@@ -46,11 +46,21 @@ class Sentinel(object):
 
 
 class MyLocal(local):
+
+    CLASS_PROP = 42
+
     def __init__(self):
         local.__init__(self)
         self.sentinel = Sentinel()
         created_sentinels.append(id(self.sentinel))
 
+    @property
+    def desc(self):
+        return self
+
+class MyLocalSubclass(MyLocal):
+    pass
+
 class WithGetattr(local):
 
     def __getattr__(self, name):
@@ -215,6 +225,22 @@ class GeventLocalTestCase(greentest.TestCase):
 
         self.assertNotEqual(a.path, b.path, 'The values in the two objects must be different')
 
+    def test_class_attr(self, kind=MyLocal):
+        mylocal = kind()
+        self.assertEqual(42, mylocal.CLASS_PROP)
+
+        mylocal.CLASS_PROP = 1
+        self.assertEqual(1, mylocal.CLASS_PROP)
+        self.assertEqual(mylocal.__dict__['CLASS_PROP'], 1)
+
+        del mylocal.CLASS_PROP
+        self.assertEqual(42, mylocal.CLASS_PROP)
+
+        self.assertIs(mylocal, mylocal.desc)
+
+    def test_class_attr_subclass(self):
+        self.test_class_attr(kind=MyLocalSubclass)
+
     def test_locals_collected_when_greenlet_dead_but_still_referenced(self):
         # https://github.com/gevent/gevent/issues/387
         import gevent