Commit 437eff3c authored by Jason Madden's avatar Jason Madden Committed by GitHub

Merge pull request #1115 from gevent/issue755

Add spawn_tree_locals, spawning_greenlet and spawning_stack to gevent.greenlet.Greenlet
parents 54ffea34 420c4d6c
......@@ -9,6 +9,7 @@ gevent.*.[ch]
src/gevent/__pycache__
src/gevent/_semaphore.c
src/gevent/local.c
src/gevent/greenlet.c
src/gevent/libev/corecext.c
src/gevent/libev/corecext.h
src/gevent/libev/_corecffi.c
......
......@@ -85,6 +85,15 @@
- Signal handling under PyPy with libuv is more reliable. See
:issue:`1112`.
- Greenlet objects now keep track of their spawning parent greenlet
and the code location that spawned them, in addition to maintaining
a "spawn tree local" mapping. Based on a proposal from PayPal and
comments by Mahmoud Hashemi and Kurt Rose. See :issue:`755` and
:pr:`1115`.
- The :mod:`gevent.greenlet` module is now compiled with Cython to
offset any performance decrease due to :issue:`755`. Please open
issues for any compatibility concerns. See :pr:`1115`.
1.3a1 (2018-01-27)
==================
......
......@@ -223,7 +223,6 @@ latex_documents = [
# prevent some stuff from showing up in docs
import socket
import gevent.socket
del gevent.Greenlet.throw
for item in gevent.socket.__all__[:]:
if getattr(gevent.socket, item) is getattr(socket, item, None):
gevent.socket.__all__.remove(item)
......@@ -36,18 +36,7 @@ generated.
.. automethod:: Greenlet.__init__
.. attribute:: Greenlet.value
Holds the value returned by the function if the greenlet has
finished successfully. Until then, or if it finished in error, ``None``.
.. tip:: Recall that a greenlet killed with the default
:class:`GreenletExit` is considered to have finished
successfully, and the ``GreenletExit`` exception will be
its value.
.. autoattribute:: Greenlet.exception
.. automethod:: Greenlet.ready
.. automethod:: Greenlet.successful
.. automethod:: Greenlet.start
......@@ -114,6 +103,8 @@ Spawn helpers
Useful general functions
========================
.. seealso:: :mod:`gevent.util`
.. function:: getcurrent()
Return the currently executing greenlet (the one that called this
......
......@@ -4,6 +4,7 @@ from __future__ import print_function
import sys
import os
import os.path
import sysconfig
# setuptools is *required* on Windows
# (https://bugs.python.org/issue23246) and for PyPy. No reason not to
......@@ -59,11 +60,28 @@ LOCAL = Extension(name="gevent.local",
depends=['src/gevent/local.pxd'])
LOCAL = cythonize1(LOCAL)
# The sysconfig dir is not enough if we're in a virtualenv
# See https://github.com/pypa/pip/issues/4610
include_dirs = [sysconfig.get_path("include")]
venv_include_dir = os.path.join(sys.prefix, 'include', 'site',
'python' + sysconfig.get_python_version())
venv_include_dir = os.path.abspath(venv_include_dir)
if os.path.exists(venv_include_dir):
include_dirs.append(venv_include_dir)
GREENLET = Extension(name="gevent.greenlet",
sources=["src/gevent/greenlet.py"],
depends=['src/gevent/greenlet.pxd'],
include_dirs=include_dirs)
GREENLET = cythonize1(GREENLET)
EXT_MODULES = [
CORE,
ARES,
SEMAPHORE,
LOCAL,
GREENLET,
]
LIBEV_CFFI_MODULE = 'src/gevent/libev/_corecffi_build.py:ffi'
......@@ -91,6 +109,7 @@ if PYPY:
setup_requires = []
EXT_MODULES.remove(CORE)
EXT_MODULES.remove(LOCAL)
EXT_MODULES.remove(GREENLET)
EXT_MODULES.remove(SEMAPHORE)
# By building the semaphore with Cython under PyPy, we get
# atomic operations (specifically, exiting/releasing), at the
......
# cython: auto_pickle=False
cimport cython
cdef extern from "greenlet/greenlet.h":
ctypedef class greenlet.greenlet [object PyGreenlet]:
pass
cdef class SpawnedLink:
cdef public object callback
@cython.final
cdef class SuccessSpawnedLink(SpawnedLink):
pass
@cython.final
cdef class FailureSpawnedLink(SpawnedLink):
pass
@cython.final
@cython.internal
cdef class _Frame:
cdef readonly object f_code
cdef readonly int f_lineno
cdef public _Frame f_back
@cython.final
@cython.locals(
previous=_Frame,
first=_Frame,
next_frame=_Frame)
cdef _Frame _extract_stack(int limit, _Frame f_back)
cdef class Greenlet(greenlet):
cdef readonly object value
cdef readonly args
cdef readonly object spawning_greenlet
cdef public dict spawn_tree_locals
cdef readonly _Frame spawning_stack
cdef list _links
cdef tuple _exc_info
cdef object _notifier
cdef object _start_event
cdef dict _kwargs
cdef str _formatted_info
cpdef bint has_links(self)
cpdef join(self, timeout=*)
cpdef bint ready(self)
cpdef bint successful(self)
cpdef rawlink(self, object callback)
cpdef str _formatinfo(self)
cdef bint __started_but_aborted(self)
cdef bint __start_cancelled_by_kill(self)
cdef bint __start_pending(self)
cdef bint __never_started_or_killed(self)
cdef bint __start_completed(self)
cdef __handle_death_before_start(self, tuple args)
cdef __cancel_start(self)
cdef _report_result(self, object result)
cdef _report_error(self, tuple exc_info)
# This is used as the target of a callback
# from the loop, and so needs to be a cpdef
cpdef _notify_links(self)
# IMapUnordered greenlets in pools need to access this
# method
cpdef _raise_exception(self)
# Declare a bunch of imports as cdefs so they can
# be accessed directly as static vars without
# doing a module global lookup. This is especially important
# for spawning greenlets.
cdef _greenlet__init__
cdef get_hub
cdef wref
cdef getcurrent
cdef Timeout
cdef dump_traceback
cdef load_traceback
cdef Waiter
cdef wait
cdef iwait
cdef reraise
cdef InvalidSwitchError
@cython.final
@cython.internal
cdef class _dummy_event:
cdef readonly bint pending
cdef readonly bint active
cpdef stop(self)
cpdef start(self, cb)
cpdef close(self)
cdef _dummy_event _cancelled_start_event
cdef _dummy_event _start_completed_event
@cython.locals(diehards=list)
cdef _killall3(list greenlets, object exception, object waiter)
cdef _killall(list greenlets, object exception)
@cython.locals(done=list)
cpdef joinall(greenlets, timeout=*, raise_error=*, count=*)
This diff is collapsed.
......@@ -9,6 +9,7 @@ from functools import partial as _functools_partial
import os
import sys
import traceback
from weakref import ref as wref
from greenlet import greenlet as RawGreenlet, getcurrent, GreenletExit
......@@ -110,6 +111,10 @@ def spawn_raw(function, *args, **kwargs):
occasionally be useful as an optimization if there are many
greenlets involved.
.. versionchanged:: 1.1a3
Verify that ``function`` is callable, raising a TypeError if not. Previously,
the spawned greenlet would have failed the first time it was switched to.
.. versionchanged:: 1.1b1
If *function* is not callable, immediately raise a :exc:`TypeError`
instead of spawning a greenlet that will raise an uncaught TypeError.
......@@ -118,12 +123,15 @@ def spawn_raw(function, *args, **kwargs):
Accept keyword arguments for ``function`` as previously (incorrectly)
documented. Note that this may incur an additional expense.
.. versionchanged:: 1.1a3
Verify that ``function`` is callable, raising a TypeError if not. Previously,
the spawned greenlet would have failed the first time it was switched to.
.. versionchanged:: 1.3a2
Populate the ``spawning_greenlet`` and ``spawn_tree_locals``
attributes of the returned greenlet.
"""
if not callable(function):
raise TypeError("function must be callable")
# The hub is always the parent.
hub = get_hub()
# The callback class object that we use to run this doesn't
......@@ -136,6 +144,19 @@ def spawn_raw(function, *args, **kwargs):
else:
g = RawGreenlet(function, hub)
hub.loop.run_callback(g.switch, *args)
# See greenlet.py's Greenlet class. We capture the cheap
# parts to maintain the tree structure, but we do not capture
# the stack because that's too expensive.
current = getcurrent()
g.spawning_greenlet = wref(current)
# See Greenlet for how trees are maintained.
try:
g.spawn_tree_locals = current.spawn_tree_locals
except AttributeError:
g.spawn_tree_locals = {}
if current.parent:
current.spawn_tree_locals = g.spawn_tree_locals
return g
......
......@@ -3,11 +3,14 @@
Low-level utilities.
"""
from __future__ import absolute_import
from __future__ import absolute_import, print_function, division
import functools
__all__ = ['wrap_errors']
__all__ = [
'wrap_errors',
'format_run_info',
]
class wrap_errors(object):
......@@ -59,7 +62,7 @@ class wrap_errors(object):
def __getattr__(self, name):
return getattr(self.__func, name)
def dump_stacks():
def format_run_info():
"""
Request information about the running threads of the current process.
......@@ -68,39 +71,82 @@ def dump_stacks():
:return: A sequence of text lines detailing the stacks of running
threads and greenlets. (One greenlet will duplicate one thread,
the current thread and greenlet.)
the current thread and greenlet.) Extra information about
:class:`gevent.greenlet.Greenlet` object will also be returned.
.. versionadded:: 1.3a1
.. versionchanged:: 1.3a2
Renamed from ``dump_stacks`` to reflect the fact that this
prints additional information about greenlets, including their
spawning stack, parent, and any spawn tree locals.
"""
dump = []
# threads
import threading # Late import this stuff because it may get monkey-patched
import traceback
import sys
import gc
lines = []
from greenlet import greenlet
_format_thread_info(lines)
_format_greenlet_info(lines)
return lines
def _format_thread_info(lines):
import threading
import sys
import traceback
threads = {th.ident: th.name for th in threading.enumerate()}
lines.append('*' * 80)
lines.append('* Threads')
thread = None
frame = None
for thread, frame in sys._current_frames().items():
dump.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
dump.append(''.join(traceback.format_stack(frame)))
dump.append('\n')
lines.append("*" * 80)
lines.append('Thread 0x%x (%s)\n' % (thread, threads.get(thread)))
lines.append(''.join(traceback.format_stack(frame)))
# We may have captured our own frame, creating a reference
# cycle, so clear it out.
del thread
del frame
del lines
del threads
def _format_greenlet_info(lines):
from greenlet import greenlet
import pprint
import traceback
import gc
# greenlets
def _noop():
return None
# if greenlet is present, let's dump each greenlet stack
# Use the gc module to inspect all objects to find the greenlets
# since there isn't a global registry
lines.append('*' * 80)
lines.append('* Greenlets')
seen_locals = set() # {id}
for ob in gc.get_objects():
if not isinstance(ob, greenlet):
continue
if not ob:
continue # not running anymore or not started
dump.append('Greenlet %s\n' % ob)
dump.append(''.join(traceback.format_stack(ob.gr_frame)))
dump.append('\n')
return dump
lines.append('*' * 80)
lines.append('Greenlet %s\n' % ob)
lines.append(''.join(traceback.format_stack(ob.gr_frame)))
spawning_stack = getattr(ob, 'spawning_stack', None)
if spawning_stack:
lines.append("Spawned at: ")
lines.append(''.join(traceback.format_stack(spawning_stack)))
parent = getattr(ob, 'spawning_greenlet', _noop)()
if parent is not None:
lines.append("Parent greenlet: %s\n" % (parent,))
spawn_tree_locals = getattr(ob, 'spawn_tree_locals', None)
if spawn_tree_locals and id(spawn_tree_locals) not in seen_locals:
seen_locals.add(id(spawn_tree_locals))
lines.append("Spawn Tree Locals:\n")
lines.append(pprint.pformat(spawn_tree_locals))
del lines
dump_stacks = format_run_info
......@@ -96,14 +96,17 @@ class TestUnlink(greentest.TestCase):
def _test_func(self, p, link):
link(dummy_test_func)
assert len(p._links) == 1, p._links
self.assertEqual(1, p.has_links())
p.unlink(dummy_test_func)
assert not p._links, p._links
self.assertEqual(0, p.has_links())
link(self.setUp)
assert len(p._links) == 1, p._links
self.assertEqual(1, p.has_links())
p.unlink(self.setUp)
assert not p._links, p._links
self.assertEqual(0, p.has_links())
p.kill()
def test_func_link(self):
......@@ -170,8 +173,7 @@ class TestReturn_link(LinksTestCase):
p = None
def cleanup(self):
while self.p._links:
self.p._links.pop()
self.p.unlink_all()
self.p = None
def test_return(self):
......@@ -374,7 +376,8 @@ class TestStuff(greentest.TestCase):
link(results.listener2)
link(results.listener3)
sleep(DELAY * 10)
assert results.results == [5], results.results
self.assertEqual([5], results.results)
def test_multiple_listeners_error_unlink_Greenlet_link(self):
p = gevent.spawn(lambda: 5)
......@@ -402,6 +405,8 @@ class A(object):
hexobj = re.compile('-?0x[0123456789abcdef]+L?', re.I)
class Subclass(gevent.Greenlet):
pass
class TestStr(greentest.TestCase):
......@@ -425,6 +430,17 @@ class TestStr(greentest.TestCase):
str_g = str_g.replace(__name__, 'module')
self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
def test_subclass(self):
g = Subclass()
str_g = hexobj.sub('X', str(g))
str_g = str_g.replace(__name__, 'module')
self.assertEqual(str_g, '<Subclass at X: _run>')
g = Subclass(None, 'question', answer=42)
str_g = hexobj.sub('X', str(g))
str_g = str_g.replace(__name__, 'module')
self.assertEqual(str_g, "<Subclass at X: _run('question', answer=42)>")
class TestJoin(AbstractGenericWaitTestCase):
......@@ -515,14 +531,14 @@ class TestBasic(greentest.TestCase):
assert g.exception is None
gevent.sleep(0.001)
assert g
assert not g.dead
assert g.started
assert not g.ready()
assert not g.successful()
assert g.value is None
assert g.exception is None
assert not link_test
self.assertTrue(g)
self.assertFalse(g.dead, g)
self.assertTrue(g.started, g)
self.assertFalse(g.ready(), g)
self.assertFalse(g.successful(), g)
self.assertIsNone(g.value, g)
self.assertIsNone(g.exception, g)
self.assertFalse(link_test)
gevent.sleep(0.02)
assert not g
......@@ -649,6 +665,29 @@ class TestBasic(greentest.TestCase):
g.join()
self.assertFalse(g.exc_info)
def test_tree_locals(self):
g = g2 = None
def func():
child = greenlet.Greenlet()
self.assertIs(child.spawn_tree_locals, getcurrent().spawn_tree_locals)
self.assertIs(child.spawning_greenlet(), getcurrent())
g = greenlet.Greenlet(func)
g2 = greenlet.Greenlet(func)
# Creating those greenlets did not give the main greenlet
# a locals dict.
self.assertFalse(hasattr(getcurrent(), 'spawn_tree_locals'),
getcurrent())
self.assertIsNot(g.spawn_tree_locals, g2.spawn_tree_locals)
g.start()
g.join()
raw = gevent.spawn_raw(func)
self.assertIsNotNone(raw.spawn_tree_locals)
self.assertIsNot(raw.spawn_tree_locals, g.spawn_tree_locals)
self.assertIs(raw.spawning_greenlet(), getcurrent())
while not raw.dead:
gevent.sleep(0.01)
class TestStart(greentest.TestCase):
......
# A greenlet that's killed before it is ever started
# should never be switched to
import gevent
import greentest
switched_to = [False, False]
def runner(i):
switched_to[i] = True
def check(g, g2):
gevent.joinall((g, g2))
assert switched_to == [False, False], switched_to
# They both have a GreenletExit as their value
assert isinstance(g.value, gevent.GreenletExit)
assert isinstance(g2.value, gevent.GreenletExit)
# They both have no reported exc_info
assert g._exc_info == (None, None, None)
assert g2._exc_info == (None, None, None)
assert g._exc_info is not type(g)._exc_info
assert g2._exc_info is not type(g2)._exc_info
switched_to[:] = [False, False]
g = gevent.spawn(runner, 0) # create but do not switch to
g2 = gevent.spawn(runner, 1) # create but do not switch to
# Using gevent.kill
gevent.kill(g)
gevent.kill(g2)
check(g, g2)
# killing directly
g = gevent.spawn(runner, 0)
g2 = gevent.spawn(runner, 1)
g.kill()
g2.kill()
check(g, g2)
# throwing
g = gevent.spawn(runner, 0)
g2 = gevent.spawn(runner, 1)
g.throw(gevent.GreenletExit)
g2.throw(gevent.GreenletExit)
check(g, g2)
# Killing with gevent.kill gets the right exception
class MyException(Exception):
pass
def catcher():
try:
while True:
gevent.sleep(0)
except Exception as e:
switched_to[0] = e
g = gevent.spawn(catcher)
g.start()
gevent.sleep()
gevent.kill(g, MyException())
gevent.sleep()
assert isinstance(switched_to[0], MyException), switched_to
class TestSwitch(greentest.TestCase):
def setUp(self):
self.switched_to = [False, False]
self.caught = None
def runner(self, i):
self.switched_to[i] = True
def check(self, g, g2):
gevent.joinall((g, g2))
self.assertEqual([False, False], self.switched_to)
# They both have a GreenletExit as their value
self.assertIsInstance(g.value, gevent.GreenletExit)
self.assertIsInstance(g2.value, gevent.GreenletExit)
# They both have no reported exc_info
self.assertIsNone(g.exc_info)
self.assertIsNone(g2.exc_info)
self.assertIsNone(g.exception)
self.assertIsNone(g2.exception)
def test_gevent_kill(self):
g = gevent.spawn(self.runner, 0) # create but do not switch to
g2 = gevent.spawn(self.runner, 1) # create but do not switch to
# Using gevent.kill
gevent.kill(g)
gevent.kill(g2)
self.check(g, g2)
def test_greenlet_kill(self):
# killing directly
g = gevent.spawn(self.runner, 0)
g2 = gevent.spawn(self.runner, 1)
g.kill()
g2.kill()
self.check(g, g2)
def test_throw(self):
# throwing
g = gevent.spawn(self.runner, 0)
g2 = gevent.spawn(self.runner, 1)
g.throw(gevent.GreenletExit)
g2.throw(gevent.GreenletExit)
self.check(g, g2)
def catcher(self):
try:
while True:
gevent.sleep(0)
except MyException as e:
self.caught = e
def test_kill_exception(self):
# Killing with gevent.kill gets the right exception
g = gevent.spawn(self.catcher)
g.start()
gevent.sleep()
gevent.kill(g, MyException())
gevent.sleep()
self.assertIsInstance(self.caught, MyException)
self.assertIsNone(g.exception, MyException)
if __name__ == '__main__':
greentest.main()
......@@ -273,7 +273,7 @@ class GeventLocalTestCase(greentest.TestCase):
self.assertEqual(count, len(deleted_sentinels))
# The links were removed as well.
self.assertEqual(list(running_greenlet._links), [])
self.assertFalse(running_greenlet.has_links())
running_greenlet = gevent.spawn(demonstrate_my_local)
......
# -*- coding: utf-8 -*-
# Copyright 2018 gevent contributes
# See LICENSE for details.
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import greentest
import gevent
from gevent import util
@greentest.skipOnPyPy("5.10.x is *very* slow formatting stacks")
class TestFormat(greentest.TestCase):
def test_basic(self):
lines = util.format_run_info()
value = '\n'.join(lines)
self.assertIn('Threads', value)
self.assertIn('Greenlets', value)
# because it's a raw greenlet, we have no data for it.
self.assertNotIn("Spawned at", value)
self.assertNotIn("Parent greenlet", value)
self.assertNotIn("Spawn Tree Locals", value)
def test_with_Greenlet(self):
def root():
gevent.getcurrent().spawn_tree_locals['a value'] = 42
g = gevent.spawn(util.format_run_info)
g.join()
return g.value
g = gevent.spawn(root)
g.join()
value = '\n'.join(g.value)
self.assertIn("Spawned at", value)
self.assertIn("Parent greenlet", value)
self.assertIn("Spawn Tree Locals", value)
if __name__ == '__main__':
greentest.main()
......@@ -13,3 +13,4 @@ test__socket_errors.py
test__socket_send_memoryview.py
test__socket_timeout.py
test__examples.py
test__issue330.py
......@@ -126,3 +126,4 @@ test_asyncore.py
test___config.py
test__destroy_default_loop.py
test__util.py
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment