Commit 0a0c16fb authored by Jason Madden's avatar Jason Madden Committed by GitHub

Merge pull request #1096 from gevent/config-issue1090

Centralize configuration in a ``gevent.config`` object.
parents 654e8302 96b35730
......@@ -46,6 +46,18 @@
:mod:`time`, much like :mod:`gevent.socket` can be imported instead
of :mod:`socket`.
- Centralize all gevent configuration in an object at
``gevent.config``, allowing for gevent to be configured through code
and not *necessarily* environment variables, and also provide a
centralized place for documentation. See :issue:`1090`.
- The new ``GEVENT_CORE_CFFI_ONLY`` environment variable has been
replaced with the pre-existing ``GEVENT_LOOP`` environment
variable. That variable may take the values ``libev-cext``,
``libev-cffi``, or ``libuv-cffi``, (or be a list in preference
order, or be a dotted name; it may also be assigned to an
object in Python code at ``gevent.config.loop``).
1.3a1 (2018-01-27)
==================
......
......@@ -45,46 +45,43 @@ prospector:
lint: prospector
test_prelim:
which ${PYTHON}
${PYTHON} --version
${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
make bench
@which ${PYTHON}
@${PYTHON} --version
@${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
@${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
@${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
@make bench
# Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44
# echo -e "travis_fold:start:${GEVENT_CORE_CFFI_ONLY}\033[33;1m${GEVENT_CORE_CFFI_ONLY}\033[0m"
# Make calls /bin/echo, which doesn't support the -e option, which is part of the bash builtin.
# we need a python script to do this, or possible the GNU make shell function
basictest: test_prelim
${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
@${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
cd src/greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config known_failures.py --quiet
${PYTHON} scripts/travis.py fold_end basictest
@${PYTHON} scripts/travis.py fold_end basictest
alltest: basictest
${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
@${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
${PYTHON} scripts/travis.py fold_end ares
${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
@${PYTHON} scripts/travis.py fold_end ares
@${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
${PYTHON} scripts/travis.py fold_end dnspython
@${PYTHON} scripts/travis.py fold_end dnspython
# In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
# text. The monkey-patched stdlib tests were specifically included here.
# However, we now always also test on AppVeyor (Windows) which only has GEVENT_FILE=thread,
# so we can save a lot of CI time by reducing the set and excluding the stdlib tests without
# losing any coverage. See the `threadfiletest` for what command used to run.
${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
@${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py test__*subprocess*.py --quiet
${PYTHON} scripts/travis.py fold_end thread
@${PYTHON} scripts/travis.py fold_end thread
threadfiletest:
cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py `grep -l subprocess test_*.py` --quiet
allbackendtest:
${PYTHON} scripts/travis.py fold_start default "Testing default backend"
GEVENT_CORE_CFFI_ONLY= GEVENTTEST_COVERAGE=1 make alltest
${PYTHON} scripts/travis.py fold_end default
@${PYTHON} scripts/travis.py fold_start default "Testing default backend"
GEVENTTEST_COVERAGE=1 make alltest
@${PYTHON} scripts/travis.py fold_end default
GEVENTTEST_COVERAGE=1 make cffibackendtest
# because we set parallel=true, each run produces new and different coverage files; they all need
# to be combined
......@@ -92,17 +89,17 @@ allbackendtest:
cffibackendtest:
${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
GEVENT_CORE_CFFI_ONLY=libuv GEVENTTEST_COVERAGE=1 make alltest
${PYTHON} scripts/travis.py fold_end libuv
${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
GEVENT_CORE_CFFI_ONLY=libev make alltest
${PYTHON} scripts/travis.py fold_end libev
@${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
@${PYTHON} scripts/travis.py fold_end libuv
@${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
GEVENT_LOOP=libev-cffi make alltest
@${PYTHON} scripts/travis.py fold_end libev
leaktest: test_prelim
${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
@${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
cd src/greentest && GEVENT_RESOLVER=thread GEVENTTEST_LEAKCHECK=1 ${PYTHON} testrunner.py --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
${PYTHON} scripts/travis.py fold_end leaktest
@${PYTHON} scripts/travis.py fold_end leaktest
bench:
${PYTHON} src/greentest/bench_sendall.py
......@@ -166,8 +163,8 @@ $(PYPY3):
develop:
${PYTHON} scripts/travis.py fold_start install "Installing gevent"
echo python is at `which $(PYTHON)`
@${PYTHON} scripts/travis.py fold_start install "Installing gevent"
@echo python is at `which $(PYTHON)`
# First install a newer pip so that it can use the wheel cache
# (only needed until travis upgrades pip to 7.x; note that the 3.5
# environment uses pip 7.1 by default)
......@@ -176,7 +173,7 @@ develop:
# disables the cache.
# We need wheel>=0.26 on Python 3.5. See previous revisions.
GEVENTSETUP_EV_VERIFY=3 ${PYTHON} -m pip install -U -r dev-requirements.txt
${PYTHON} scripts/travis.py fold_end install
@${PYTHON} scripts/travis.py fold_end install
test-py27: $(PY27)
PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest allbackendtest
......@@ -200,7 +197,13 @@ test-pypy3: $(PYPY3)
PYTHON=$(PYPY3) PATH=$(BUILD_RUNTIMES)/versions/pypy3.5_5101/bin:$(PATH) make develop basictest
test-py27-noembed: $(PY27)
@python2.7.14 scripts/travis.py fold_start conf_libev "Configuring libev"
cd deps/libev && ./configure --disable-dependency-tracking && make
@python2.7.14 scripts/travis.py fold_end conf_libev
@python2.7.14 scripts/travis.py fold_start conf_cares "Configuring cares"
cd deps/c-ares && ./configure --disable-dependency-tracking && make
@python2.7.14 scripts/travis.py fold_end conf_cares
@python2.7.14 scripts/travis.py fold_start conf_libuv "Configuring libuv"
cd deps/libuv && ./autogen.sh && ./configure --disable-static && make
CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
@python2.7.14 scripts/travis.py fold_end conf_libuv
CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_LOOP=libev-cext PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
......@@ -26,7 +26,7 @@ environment:
PYTHON_VERSION: "3.6.x" # currently 3.6.0
PYTHON_ARCH: "64"
PYTHON_EXE: python
GEVENT_CORE_CFFI_ONLY: libuv
GEVENT_LOOP: libuv
- PYTHON: "C:\\Python27-x64"
PYTHON_VERSION: "2.7.x" # currently 2.7.13
......
......@@ -87,7 +87,7 @@ release = version
exclude_trees = ['_build']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
default_role = 'obj'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True
......
......@@ -191,3 +191,9 @@ Timeouts
:special-members: __enter__, __exit__
.. autofunction:: with_timeout
Configuration
=============
.. autoclass:: gevent._config.Config
......@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet
to provide a high-level synchronous API on top of libev event loop.
See http://www.gevent.org/ for the documentation.
.. versionchanged:: 1.3a2
Add the `config` object.
"""
from __future__ import absolute_import
......@@ -43,6 +46,8 @@ __all__ = [
'reinit',
'getswitchinterval',
'setswitchinterval',
# Added in 1.3a2
'config',
]
......@@ -72,11 +77,14 @@ except AttributeError:
global _switchinterval
_switchinterval = interval
from gevent._config import config
from gevent.hub import get_hub, iwait, wait
from gevent.greenlet import Greenlet, joinall, killall
joinall = joinall # export for pylint
spawn = Greenlet.spawn
spawn_later = Greenlet.spawn_later
#: The singleton configuration object for gevent.
config = config
from gevent.timeout import Timeout, with_timeout
from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit
......
This diff is collapsed.
......@@ -10,14 +10,17 @@ import signal as signalmodule
import functools
import warnings
from gevent._config import config
try:
from tracemalloc import get_object_traceback
def tracemalloc(init):
# PYTHONTRACEMALLOC env var controls this.
# PYTHONTRACEMALLOC env var controls this on Python 3.
return init
except ImportError: # Python < 3.4
if os.getenv('PYTHONTRACEMALLOC'):
if config.trace_malloc:
# Use the same env var to turn this on for Python 2
import traceback
......
from __future__ import absolute_import, print_function, division
try:
from errno import EBADF
except ImportError:
EBADF = 9
import os
from io import TextIOWrapper
import functools
import sys
from gevent.hub import get_hub
from gevent._compat import integer_types
from gevent._compat import reraise
from gevent.lock import Semaphore, DummySemaphore
class cancel_wait_ex(IOError):
......@@ -136,3 +146,130 @@ class FileObjectBase(object):
def __exit__(self, *args):
self.close()
class FileObjectBlock(FileObjectBase):
def __init__(self, fobj, *args, **kwargs):
closefd = kwargs.pop('close', True)
if kwargs:
raise TypeError('Unexpected arguments: %r' % kwargs.keys())
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectBlock does not support close=False on an fd.')
fobj = os.fdopen(fobj, *args)
super(FileObjectBlock, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
fobj.close()
class FileObjectThread(FileObjectBase):
"""
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
.. caution::
Attempting to change the threadpool or lock of an existing FileObjectThread
has undefined consequences.
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
"""
:param fobj: The underlying file-like object to wrap, or an integer fileno
that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
in any particular order, only that no two operations will be attempted at the
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
file operations with an external resource.
:keyword bool close: If True (the default) then when this object is closed,
the underlying object is closed as well.
"""
closefd = close
self.threadpool = threadpool or get_hub().threadpool
self.lock = lock
if self.lock is True:
self.lock = Semaphore()
elif not self.lock:
self.lock = DummySemaphore()
if not hasattr(self.lock, '__enter__'):
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectThread does not support close=False on an fd.')
if mode is None:
assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
fobj = os.fdopen(fobj)
else:
fobj = os.fdopen(fobj, mode, bufsize)
self.__io_holder = [fobj] # signal for _wrap_method
super(FileObjectThread, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
self.__io_holder[0] = None # for _wrap_method
try:
with self.lock:
self.threadpool.apply(fobj.flush)
finally:
if closefd:
# Note that we're not taking the lock; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged by the pool
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _do_delegate_methods(self):
super(FileObjectThread, self)._do_delegate_methods()
if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
self.read1 = self.read
self.__io_holder[0] = self._io
def _extra_repr(self):
return ' threadpool=%r' % (self.threadpool,)
def __iter__(self):
return self
def next(self):
line = self.readline()
if line:
return line
raise StopIteration
__next__ = next
def _wrap_method(self, method):
# NOTE: We are careful to avoid introducing a refcycle
# within self. Our wrapper cannot refer to self.
io_holder = self.__io_holder
lock = self.lock
threadpool = self.threadpool
@functools.wraps(method)
def thread_method(*args, **kwargs):
if io_holder[0] is None:
# This is different than FileObjectPosix, etc,
# because we want to save the expensive trip through
# the threadpool.
raise FileObjectClosed()
with lock:
return threadpool.apply(method, args, kwargs)
return thread_method
# Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
"""
Deprecated; this does not reflect all the possible options
and its interface varies.
.. versionchanged:: 1.3a2
Deprecated.
"""
from __future__ import absolute_import
import os
import sys
from gevent._config import config
from gevent._util import copy_globals
try:
if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
raise ImportError("Not attempting corecext")
from gevent.libev import corecext as _core
except ImportError:
if os.environ.get('GEVENT_CORE_CEXT_ONLY'):
raise
# CFFI/PyPy
lib = os.environ.get('GEVENT_CORE_CFFI_ONLY')
if lib == 'libuv':
from gevent.libuv import loop as _core
else:
try:
from gevent.libev import corecffi as _core
except ImportError:
from gevent.libuv import loop as _core
_core = sys.modules[config.loop.__module__]
copy_globals(_core, globals())
......
......@@ -35,31 +35,12 @@ Classes
"""
from __future__ import absolute_import
import functools
import sys
import os
from gevent._fileobjectcommon import FileObjectClosed
from gevent._fileobjectcommon import FileObjectBase
from gevent.hub import get_hub
from gevent._compat import integer_types
from gevent._compat import reraise
from gevent.lock import Semaphore, DummySemaphore
PYPY = hasattr(sys, 'pypy_version_info')
if hasattr(sys, 'exc_clear'):
def _exc_clear():
sys.exc_clear()
else:
def _exc_clear():
return
from gevent._config import config
__all__ = [
'FileObjectPosix',
'FileObjectThread',
'FileObjectBlock',
'FileObject',
]
......@@ -71,149 +52,10 @@ else:
del fcntl
from gevent._fileobjectposix import FileObjectPosix
from gevent._fileobjectcommon import FileObjectThread
from gevent._fileobjectcommon import FileObjectBlock
class FileObjectThread(FileObjectBase):
"""
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
.. caution::
Attempting to change the threadpool or lock of an existing FileObjectThread
has undefined consequences.
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
"""
:param fobj: The underlying file-like object to wrap, or an integer fileno
that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
in any particular order, only that no two operations will be attempted at the
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
file operations with an external resource.
:keyword bool close: If True (the default) then when this object is closed,
the underlying object is closed as well.
"""
closefd = close
self.threadpool = threadpool or get_hub().threadpool
self.lock = lock
if self.lock is True:
self.lock = Semaphore()
elif not self.lock:
self.lock = DummySemaphore()
if not hasattr(self.lock, '__enter__'):
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectThread does not support close=False on an fd.')
if mode is None:
assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
fobj = os.fdopen(fobj)
else:
fobj = os.fdopen(fobj, mode, bufsize)
self.__io_holder = [fobj] # signal for _wrap_method
super(FileObjectThread, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
self.__io_holder[0] = None # for _wrap_method
try:
with self.lock:
self.threadpool.apply(fobj.flush)
finally:
if closefd:
# Note that we're not taking the lock; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged by the pool
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _do_delegate_methods(self):
super(FileObjectThread, self)._do_delegate_methods()
if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
self.read1 = self.read
self.__io_holder[0] = self._io
def _extra_repr(self):
return ' threadpool=%r' % (self.threadpool,)
def __iter__(self):
return self
def next(self):
line = self.readline()
if line:
return line
raise StopIteration
__next__ = next
def _wrap_method(self, method):
# NOTE: We are careful to avoid introducing a refcycle
# within self. Our wrapper cannot refer to self.
io_holder = self.__io_holder
lock = self.lock
threadpool = self.threadpool
@functools.wraps(method)
def thread_method(*args, **kwargs):
if io_holder[0] is None:
# This is different than FileObjectPosix, etc,
# because we want to save the expensive trip through
# the threadpool.
raise FileObjectClosed()
with lock:
return threadpool.apply(method, args, kwargs)
return thread_method
try:
FileObject = FileObjectPosix
except NameError:
FileObject = FileObjectThread
class FileObjectBlock(FileObjectBase):
def __init__(self, fobj, *args, **kwargs):
closefd = kwargs.pop('close', True)
if kwargs:
raise TypeError('Unexpected arguments: %r' % kwargs.keys())
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectBlock does not support close=False on an fd.')
fobj = os.fdopen(fobj, *args)
super(FileObjectBlock, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
fobj.close()
config = os.environ.get('GEVENT_FILE')
if config:
klass = {'thread': 'gevent.fileobject.FileObjectThread',
'posix': 'gevent.fileobject.FileObjectPosix',
'block': 'gevent.fileobject.FileObjectBlock'}.get(config, config)
if klass.startswith('gevent.fileobject.'):
FileObject = globals()[klass.split('.', 2)[-1]]
else:
from gevent.hub import _import
FileObject = _import(klass)
del klass
# None of the possible objects can live in this module because
# we would get an import cycle and the config couldn't be set from code.
FileObject = config.fileobject
......@@ -26,6 +26,7 @@ __all__ = [
'Waiter',
]
from gevent._config import config
from gevent._compat import string_types
from gevent._compat import xrange
from gevent._util import _NONE
......@@ -389,89 +390,29 @@ def set_hub(hub):
_threadlocal.hub = hub
def _import(path):
# pylint:disable=too-many-branches
if isinstance(path, list):
if not path:
raise ImportError('Cannot import from empty list: %r' % (path, ))
for item in path[:-1]:
try:
return _import(item)
except ImportError:
pass
return _import(path[-1])
if not isinstance(path, string_types):
return path
if '.' not in path:
raise ImportError("Cannot import %r (required format: [path/][package.]module.class)" % path)
if '/' in path:
# This is dangerous, subject to race conditions, and
# may not work properly for things like namespace packages
import warnings
warnings.warn("Absolute paths are deprecated. Please put the package "
"on sys.path first",
DeprecationWarning)
package_path, path = path.rsplit('/', 1)
sys.path = [package_path] + sys.path
else:
package_path = None
try:
module, item = path.rsplit('.', 1)
x = __import__(module)
for attr in path.split('.')[1:]:
oldx = x
x = getattr(x, attr, _NONE)
if x is _NONE:
raise ImportError('Cannot import %r from %r' % (attr, oldx))
return x
finally:
if '/' in path:
try:
sys.path.remove(package_path)
except ValueError:
pass
def config(default, envvar):
def _config(default, envvar):
result = os.environ.get(envvar) or default # absolute import gets confused pylint: disable=no-member
if isinstance(result, string_types):
return result.split(',')
return result
def resolver_config(default, envvar):
result = config(default, envvar)
return [_resolvers.get(x, x) for x in result]
_resolvers = {
'ares': 'gevent.resolver.ares.Resolver',
'thread': 'gevent.resolver.thread.Resolver',
'block': 'gevent.resolver.blocking.Resolver',
'dnspython': 'gevent.resolver.dnspython.Resolver',
}
_DEFAULT_LOOP_CLASS = 'gevent.core.loop'
class Hub(RawGreenlet):
"""A greenlet that runs the event loop.
"""
A greenlet that runs the event loop.
It is created automatically by :func:`get_hub`.
**Switching**
.. rubric:: Switching
Every time this greenlet (i.e., the event loop) is switched *to*, if
the current greenlet has a ``switch_out`` method, it will be called. This
allows a greenlet to take some cleanup actions before yielding control. This method
should not call any gevent blocking functions.
Every time this greenlet (i.e., the event loop) is switched *to*,
if the current greenlet has a ``switch_out`` method, it will be
called. This allows a greenlet to take some cleanup actions before
yielding control. This method should not call any gevent blocking
functions.
"""
#: If instances of these classes are raised into the event loop,
......@@ -483,37 +424,8 @@ class Hub(RawGreenlet):
#: do not get logged/printed when raised by the event loop.
NOT_ERROR = (GreenletExit, SystemExit)
loop_class = config(_DEFAULT_LOOP_CLASS, 'GEVENT_LOOP')
# For the standard class, go ahead and import it when this class
# is defined. This is no loss of generality because the envvar is
# only read when this class is defined, and we know that the
# standard class will be available. This can solve problems with
# the class being imported from multiple threads at once, leading
# to one of the imports failing. Only do this for the object we
# need in the constructor, as the rest of the factories are
# themselves handled lazily. See #687. (People using a custom loop_class
# can probably manage to get_hub() from the main thread or otherwise import
# that loop_class themselves.)
if loop_class == [_DEFAULT_LOOP_CLASS]:
loop_class = [_import(loop_class)]
resolver_class = [
'gevent.resolver.thread.Resolver',
'gevent.resolver.dnspython.Resolver',
'gevent.resolver.ares.Resolver',
'gevent.resolver.blacking.Resolver',
]
#: The class or callable object, or the name of a factory function or class,
#: that will be used to create :attr:`resolver`. By default, configured according to
#: :doc:`dns`. If a list, a list of objects in preference order.
resolver_class = resolver_config(resolver_class, 'GEVENT_RESOLVER')
threadpool_class = config('gevent.threadpool.ThreadPool', 'GEVENT_THREADPOOL')
backend = config(None, 'GEVENT_BACKEND')
threadpool_size = 10
# using pprint.pformat can override custom __repr__ methods on dict/list
# subclasses, which can be a security concern
format_context = 'pprint.saferepr'
threadpool_size = 10
def __init__(self, loop=None, default=None):
......@@ -530,13 +442,21 @@ class Hub(RawGreenlet):
else:
if default is None and get_ident() != MAIN_THREAD:
default = False
loop_class = _import(self.loop_class)
if loop is None:
loop = self.backend
self.loop = loop_class(flags=loop, default=default)
self.loop = self.loop_class(flags=loop, default=default) # pylint:disable=not-callable
self._resolver = None
self._threadpool = None
self.format_context = _import(self.format_context)
self.format_context = config.format_context
@property
def loop_class(self):
return config.loop
@property
def backend(self):
return config.libev_backend
def __repr__(self):
if self.loop is None:
......@@ -787,11 +707,13 @@ class Hub(RawGreenlet):
if _threadlocal.hub is self:
_threadlocal.hub = None
@property
def resolver_class(self):
return config.resolver
def _get_resolver(self):
if self._resolver is None:
if self.resolver_class is not None:
self.resolver_class = _import(self.resolver_class)
self._resolver = self.resolver_class(hub=self)
self._resolver = self.resolver_class(hub=self) # pylint:disable=not-callable
return self._resolver
def _set_resolver(self, value):
......@@ -802,11 +724,15 @@ class Hub(RawGreenlet):
resolver = property(_get_resolver, _set_resolver, _del_resolver)
@property
def threadpool_class(self):
return config.threadpool
def _get_threadpool(self):
if self._threadpool is None:
if self.threadpool_class is not None:
self.threadpool_class = _import(self.threadpool_class)
self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
# pylint:disable=not-callable
self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
return self._threadpool
def _set_threadpool(self, value):
......
......@@ -46,6 +46,7 @@ from __future__ import absolute_import
import os
import sys
from gevent.hub import get_hub, reinit
from gevent._config import config
from gevent._compat import PY3
from gevent._util import copy_globals
import errno
......@@ -418,7 +419,7 @@ if hasattr(os, 'fork'):
__extensions__.append('forkpty_and_watch')
# Watch children by default
if not os.getenv('GEVENT_NOWAITPID'):
if not config.disable_watch_children:
# Broken out into separate functions instead of simple name aliases
# for documentation purposes.
def fork(*args, **kwargs):
......
......@@ -27,6 +27,9 @@ from gevent.socket import SOCK_DGRAM
from gevent.socket import SOCK_RAW
from gevent.socket import AI_NUMERICHOST
from gevent._config import config
from gevent._config import AresSettingMixin
from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
from . import _lookup_port as lookup_port
from . import _resolve_special
......@@ -84,12 +87,11 @@ class Resolver(AbstractResolver):
hub = get_hub()
self.hub = hub
if use_environ:
for key in os.environ:
if key.startswith('GEVENTARES_'):
name = key[11:].lower()
if name:
value = os.environ[key]
kwargs.setdefault(name, value)
for setting in config.settings.values():
if isinstance(setting, AresSettingMixin):
value = setting.get()
if value is not None:
kwargs.setdefault(setting.kwarg_name, value)
self.ares = self.ares_class(hub.loop, **kwargs)
self.pid = os.getpid()
self.params = kwargs
......
......@@ -253,7 +253,9 @@ _is_ipv4_addr = _is_addr
def _is_ipv6_addr(host):
# Return True if host is a valid IPv6 address
host = host.split('%', 1)[0] if host else host
if host:
s = '%' if isinstance(host, str) else b'%'
host = host.split(s, 1)[0]
return _is_addr(host, dns.ipv6.inet_aton)
class HostsFile(object):
......
......@@ -11,7 +11,6 @@ from gevent.hub import get_hub
from gevent.hub import sleep as _g_sleep
from gevent._compat import integer_types
from gevent._compat import iteritems
from gevent._compat import itervalues
from gevent._util import copy_globals
from gevent._util import _NONE
......@@ -200,7 +199,11 @@ if original_poll is not None:
.. versionadded:: 1.1b1
"""
def __init__(self):
self.fds = {} # {int -> watcher}
# {int -> flags}
# We can't keep watcher objects in here because people commonly
# just drop the poll object when they're done, without calling
# unregister(). dnspython does this.
self.fds = {}
self.loop = get_hub().loop
def register(self, fd, eventmask=_NONE):
......@@ -216,12 +219,7 @@ if original_poll is not None:
# that. Should we raise an error?
fileno = get_fileno(fd)
if fileno in self.fds:
self.fds[fileno].close()
watcher = self.loop.io(fileno, flags)
watcher.priority = self.loop.MAXPRI
self.fds[fileno] = watcher
self.fds[fileno] = flags
def modify(self, fd, eventmask):
self.register(fd, eventmask)
......@@ -234,16 +232,23 @@ if original_poll is not None:
File descriptors that are closed are reported with POLLNVAL.
"""
result = PollResult()
watchers = []
io = self.loop.io
MAXPRI = self.loop.MAXPRI
try:
for fd, watcher in iteritems(self.fds):
for fd, flags in iteritems(self.fds):
watcher = io(fd, flags)
watchers.append(watcher)
watcher.priority = MAXPRI
watcher.start(result.add_event, fd, pass_events=True)
if timeout is not None and timeout > -1:
timeout /= 1000.0
result.event.wait(timeout=timeout)
return list(result.events)
finally:
for awatcher in itervalues(self.fds):
for awatcher in watchers:
awatcher.stop()
awatcher.close()
def unregister(self, fd):
"""
......@@ -254,8 +259,6 @@ if original_poll is not None:
library. Previously gevent did nothing.
"""
fileno = get_fileno(fd)
io = self.fds[fileno]
io.close()
del self.fds[fileno]
del original_poll
......@@ -58,6 +58,7 @@ from greentest.sysinfo import CONN_ABORTED_ERRORS
from greentest.skipping import skipOnWindows
from greentest.skipping import skipOnAppVeyor
from greentest.skipping import skipOnCI
from greentest.skipping import skipOnPyPy3OnCI
from greentest.skipping import skipOnPyPy
from greentest.skipping import skipOnPyPy3
......@@ -100,8 +101,6 @@ from greentest.openfiles import get_number_open_files
from greentest.openfiles import get_open_files
from greentest.testcase import TestCase
from greentest.timing import AbstractGenericGetTestCase as GenericGetTestCase
from greentest.timing import AbstractGenericWaitTestCase as GenericWaitTestCase
from greentest.modules import walk_modules
......
......@@ -32,43 +32,44 @@ def ignores_leakcheck(func):
func.ignore_leakcheck = True
return func
# Some builtin things that we ignore
IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType)
try:
callback_kind = gevent.core.callback
except AttributeError:
# Must be using FFI.
from gevent._ffi.callback import callback as callback_kind
def _type_hist():
d = collections.defaultdict(int)
for x in gc.get_objects():
k = type(x)
if k in IGNORED_TYPES:
continue
if k == callback_kind and x.callback is None and x.args is None:
# these represent callbacks that have been stopped, but
# the event loop hasn't cycled around to run them. The only
# known cause of this is killing greenlets before they get a chance
# to run for the first time.
continue
d[k] += 1
return d
def _report_diff(a, b):
diff_lines = []
for k, v in sorted(a.items(), key=lambda i: i[0].__name__):
if b[k] != v:
diff_lines.append("%s: %s != %s" % (k, v, b[k]))
if not diff_lines:
return None
diff = '\n'.join(diff_lines)
return diff
def wrap_refcount(method):
if getattr(method, 'ignore_leakcheck', False):
return method
# Some builtin things that we ignore
IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType)
try:
callback_kind = gevent.core.callback
except AttributeError:
# Must be using FFI.
from gevent._ffi.callback import callback as callback_kind
def type_hist():
d = collections.defaultdict(int)
for x in gc.get_objects():
k = type(x)
if k in IGNORED_TYPES:
continue
if k == callback_kind and x.callback is None and x.args is None:
# these represent callbacks that have been stopped, but
# the event loop hasn't cycled around to run them. The only
# known cause of this is killing greenlets before they get a chance
# to run for the first time.
continue
d[k] += 1
return d
def report_diff(a, b):
diff_lines = []
for k, v in sorted(a.items(), key=lambda i: i[0].__name__):
if b[k] != v:
diff_lines.append("%s: %s != %s" % (k, v, b[k]))
if not diff_lines:
return None
diff = '\n'.join(diff_lines)
return diff
@wraps(method)
def wrapper(self, *args, **kwargs): # pylint:disable=too-many-branches
......@@ -78,25 +79,33 @@ def wrap_refcount(method):
deltas = []
d = None
gc.disable()
# The very first time we are called, we have already been
# self.setUp() by the test runner, so we don't need to do it again.
needs_setUp = False
try:
while True:
# Grab current snapshot
hist_before = type_hist()
hist_before = _type_hist()
d = sum(hist_before.values())
self.setUp()
if needs_setUp:
self.setUp()
self.skipTearDown = False
try:
method(self, *args, **kwargs)
finally:
self.tearDown()
self.skipTearDown = True
needs_setUp = True
# Grab post snapshot
if 'urlparse' in sys.modules:
sys.modules['urlparse'].clear_cache()
if 'urllib.parse' in sys.modules:
sys.modules['urllib.parse'].clear_cache()
hist_after = type_hist()
hist_after = _type_hist()
d = sum(hist_after.values()) - d
deltas.append(d)
......@@ -119,7 +128,7 @@ def wrap_refcount(method):
elif len(deltas) >= 4 and sum(deltas[-4:]) == 0:
break
elif len(deltas) >= 3 and deltas[-1] > 0 and deltas[-1] == deltas[-2] and deltas[-2] == deltas[-3]:
diff = report_diff(hist_before, hist_after)
diff = _report_diff(hist_before, hist_after)
raise AssertionError('refcount increased by %r\n%s' % (deltas, diff))
# OK, we don't know for sure yet. Let's search for more
if sum(deltas[-3:]) <= 0 or sum(deltas[-4:]) <= 0 or deltas[-4:].count(0) >= 2:
......@@ -128,9 +137,10 @@ def wrap_refcount(method):
else:
limit = 7
if len(deltas) >= limit:
raise AssertionError('refcount increased by %r\n%s' % (deltas, report_diff(hist_before, hist_after)))
raise AssertionError('refcount increased by %r\n%s'
% (deltas,
_report_diff(hist_before, hist_after)))
finally:
gc.enable()
self.skipTearDown = True
return wrapper
......@@ -23,9 +23,11 @@ from greentest.sysinfo import PY3
from greentest.sysinfo import PYPY
from greentest.sysinfo import WIN
from greentest.sysinfo import LIBUV
from greentest.sysinfo import OSX
from greentest.sysinfo import RUNNING_ON_TRAVIS
from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
from greentest.sysinfo import RESOLVER_ARES
# Travis is slow and overloaded; Appveyor used to be faster, but
......@@ -60,6 +62,11 @@ if RUNNING_ON_TRAVIS:
# connected to with the same error.
DEFAULT_BIND_ADDR = '127.0.0.1'
if RESOLVER_ARES and OSX:
# Ares likes to raise "malformed domain name" on '', at least
# on OS X
DEFAULT_BIND_ADDR = '127.0.0.1'
# For in-process sockets
DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
......
......@@ -23,13 +23,12 @@ import unittest
from greentest import sysinfo
def _identity(f):
return f
def _do_not_skip(reason):
assert reason
def dec(f):
return f
return dec
return _identity
if sysinfo.WIN:
......@@ -50,6 +49,11 @@ if sysinfo.RUNNING_ON_APPVEYOR:
else:
skipOnAppVeyor = _do_not_skip
if sysinfo.RUNNING_ON_CI:
skipOnCI = unittest.skip
else:
skipOnCI = _do_not_skip
if sysinfo.PYPY3 and sysinfo.RUNNING_ON_CI:
# Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
skipOnPyPy3OnCI = unittest.skip
......
......@@ -30,8 +30,8 @@ LINUX = sys.platform.startswith('linux')
OSX = sys.platform == 'darwin'
# XXX: Formalize this better
LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' or (PYPY and WIN) or hasattr(gevent.core, 'libuv')
CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY
LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '')
if '--debug-greentest' in sys.argv:
sys.argv.remove('--debug-greentest')
......
......@@ -95,7 +95,9 @@ class TestCaseMetaClass(type):
if sysinfo.RUN_LEAKCHECKS and timeout is not None:
timeout *= 6
check_totalrefcount = _get_class_attr(classDict, bases, 'check_totalrefcount', True)
error_fatal = _get_class_attr(classDict, bases, 'error_fatal', True)
uses_handle_error = _get_class_attr(classDict, bases, 'uses_handle_error', True)
# Python 3: must copy, we mutate the classDict. Interestingly enough,
# it doesn't actually error out, but under 3.6 we wind up wrapping
# and re-wrapping the same items over and over and over.
......@@ -108,7 +110,8 @@ class TestCaseMetaClass(type):
error_fatal = getattr(value, 'error_fatal', error_fatal)
if error_fatal:
value = errorhandler.wrap_error_fatal(value)
value = errorhandler.wrap_restore_handle_error(value)
if uses_handle_error:
value = errorhandler.wrap_restore_handle_error(value)
if check_totalrefcount and sysinfo.RUN_LEAKCHECKS:
value = leakcheck.wrap_refcount(value)
classDict[key] = value
......@@ -122,6 +125,7 @@ class TestCase(TestCaseMetaClass("NewBase", (TimeAssertMixin, BaseTestCase,), {}
switch_expected = 'default'
error_fatal = True
uses_handle_error = True
close_on_teardown = ()
def run(self, *args, **kwargs):
......
......@@ -58,6 +58,7 @@ class _DelayWaitMixin(object):
# otherwise it's the raw number
seconds = getattr(timeout, 'seconds', timeout)
gevent.get_hub().loop.update_now()
start = time.time()
try:
result = self.wait(timeout)
......
# Copyright 2018 gevent contributors. See LICENSE for details.
import os
import unittest
import sys
from gevent import _config
class TestResolver(unittest.TestCase):
old_resolver = None
def setUp(self):
if 'GEVENT_RESOLVER' in os.environ:
self.old_resolver = os.environ['GEVENT_RESOLVER']
del os.environ['GEVENT_RESOLVER']
def tearDown(self):
if self.old_resolver:
os.environ['GEVENT_RESOLVER'] = self.old_resolver
def test_key(self):
self.assertEqual(_config.Resolver.environment_key, 'GEVENT_RESOLVER')
def test_default(self):
from gevent.resolver.thread import Resolver
conf = _config.Resolver()
self.assertEqual(conf.get(), Resolver)
def test_env(self):
from gevent.resolver.blocking import Resolver
os.environ['GEVENT_RESOLVER'] = 'foo,bar,block,dnspython'
conf = _config.Resolver()
self.assertEqual(conf.get(), Resolver)
os.environ['GEVENT_RESOLVER'] = 'dnspython'
# The existing value is unchanged
self.assertEqual(conf.get(), Resolver)
# A new object reflects it
conf = _config.Resolver()
from gevent.resolver.dnspython import Resolver as DResolver
self.assertEqual(conf.get(), DResolver)
def test_set_str_long(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set('gevent.resolver.blocking.Resolver')
self.assertEqual(conf.get(), Resolver)
def test_set_str_short(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set('block')
self.assertEqual(conf.get(), Resolver)
def test_set_class(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set(Resolver)
self.assertEqual(conf.get(), Resolver)
def test_set_through_config(self):
from gevent.resolver.thread import Resolver as Default
from gevent.resolver.blocking import Resolver
conf = _config.Config()
self.assertEqual(conf.resolver, Default)
conf.resolver = 'block'
self.assertEqual(conf.resolver, Resolver)
class TestFunctions(unittest.TestCase):
def test_validate_bool(self):
self.assertTrue(_config.validate_bool('on'))
self.assertTrue(_config.validate_bool('1'))
self.assertFalse(_config.validate_bool('off'))
self.assertFalse(_config.validate_bool('0'))
self.assertFalse(_config.validate_bool(''))
with self.assertRaises(ValueError):
_config.validate_bool(' hmm ')
def test_validate_invalid(self):
with self.assertRaises(ValueError):
_config.validate_invalid(self)
class TestConfig(unittest.TestCase):
def test__dir__(self):
self.assertEqual(sorted(_config.config.settings),
sorted(dir(_config.config)))
def test_getattr(self):
# Bypass the property that might be set here
self.assertIsNotNone(_config.config.__getattr__('resolver'))
def test__getattr__invalid(self):
with self.assertRaises(AttributeError):
getattr(_config.config, 'no_such_setting')
def test_set_invalid(self):
with self.assertRaises(AttributeError):
_config.config.set('no such setting', True)
class TestImportableSetting(unittest.TestCase):
assertRaisesRegex = getattr(unittest.TestCase, 'assertRaisesRegex',
unittest.TestCase.assertRaisesRegexp)
def test_empty_list(self):
i = _config.ImportableSetting()
with self.assertRaisesRegex(ImportError,
"Cannot import from empty list"):
i._import([])
def test_path(self):
import warnings
i = _config.ImportableSetting()
path = list(sys.path)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
with self.assertRaisesRegex(ImportError,
"Cannot import 'no_such_module'"):
i._import('foo/bar/gevent.no_such_module')
# We restored the path
self.assertEqual(path, sys.path)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, DeprecationWarning)
self.assertIn('Absolute paths', str(w[0].message))
def test_non_string(self):
i = _config.ImportableSetting()
self.assertIs(i._import(self), self)
if __name__ == '__main__':
unittest.main()
......@@ -9,8 +9,8 @@ from unittest import SkipTest
import socket
import ssl
import greentest
from greentest import DEFAULT_XPC_SOCKET_TIMEOUT
from greentest import main
from greentest import util
from greentest import params
......@@ -101,6 +101,7 @@ class Test_wsgiserver_ssl(Test_wsgiserver):
ssl_ctx = ssl._create_unverified_context()
@greentest.skipOnCI("Timing issues sometimes lead to a connection refused")
class Test_webproxy(Test_wsgiserver):
server = 'webproxy.py'
......@@ -136,4 +137,4 @@ class Test_webproxy(Test_wsgiserver):
if __name__ == '__main__':
main()
greentest.main()
......@@ -105,9 +105,10 @@ class TestTimers(greentest.TestCase):
gevent.sleep(0.02)
gevent.spawn(func)
assert lst == [1], lst
gevent.sleep(0.03)
assert lst == [], lst
self.assertEqual(lst, [1])
gevent.sleep(0.1)
self.assertEqual(lst, [])
def test_spawn_is_not_cancelled(self):
lst = [1]
......@@ -116,8 +117,8 @@ class TestTimers(greentest.TestCase):
gevent.spawn(lst.pop)
# exiting immediately, but self.lst.pop must be called
gevent.spawn(func)
gevent.sleep(0.01)
assert lst == [], lst
gevent.sleep(0.1)
self.assertEqual(lst, [])
if __name__ == '__main__':
......
from __future__ import print_function
from gevent import core
from gevent import config
from greentest import TestCase
from greentest import main
......@@ -9,15 +9,23 @@ from greentest.sysinfo import CFFI_BACKEND
class Test(TestCase):
__timeout__ = LARGE_TIMEOUT
repeat = 0
def setUp(self):
self.called = []
self.loop = core.loop(default=True)
self.loop = config.loop(default=False)
self.timer = self.loop.timer(0.001, repeat=self.repeat)
def tearDown(self):
def cleanup(self):
# cleanup instead of tearDown to cooperate well with
# leakcheck.py
self.timer.close()
# cycle the loop so libuv close callbacks fire
self.loop.run()
self.loop.destroy()
self.loop = None
self.timer = None
def f(self, x=None):
self.called.append(1)
......@@ -67,6 +75,8 @@ class TestAgain(Test):
self.assertEqual(x.args, (x,))
# XXX: On libev, this takes 1 second. On libuv,
# it takes the expected time.
self.loop.run()
self.assertEqual(self.called, [1])
......
......@@ -6,11 +6,13 @@ from gevent.event import Event, AsyncResult
import greentest
from greentest.skipping import skipUnderCoverage
from greentest.six import xrange
from greentest.timing import AbstractGenericGetTestCase
from greentest.timing import AbstractGenericWaitTestCase
DELAY = 0.01
class TestEventWait(greentest.GenericWaitTestCase):
class TestEventWait(AbstractGenericWaitTestCase):
def wait(self, timeout):
Event().wait(timeout=timeout)
......@@ -19,7 +21,7 @@ class TestEventWait(greentest.GenericWaitTestCase):
str(Event())
class TestWaitEvent(greentest.GenericWaitTestCase):
class TestWaitEvent(AbstractGenericWaitTestCase):
def wait(self, timeout):
gevent.wait([Event()], timeout=timeout)
......@@ -58,19 +60,19 @@ class TestWaitEvent(greentest.GenericWaitTestCase):
gevent.spawn(waiter).join()
class TestAsyncResultWait(greentest.GenericWaitTestCase):
class TestAsyncResultWait(AbstractGenericWaitTestCase):
def wait(self, timeout):
AsyncResult().wait(timeout=timeout)
class TestWaitAsyncResult(greentest.GenericWaitTestCase):
class TestWaitAsyncResult(AbstractGenericWaitTestCase):
def wait(self, timeout):
gevent.wait([AsyncResult()], timeout=timeout)
class TestAsyncResultGet(greentest.GenericGetTestCase):
class TestAsyncResultGet(AbstractGenericGetTestCase):
def wait(self, timeout):
AsyncResult().get(timeout=timeout)
......@@ -232,5 +234,8 @@ class TestWait_count2(TestWait):
count = 2
del AbstractGenericGetTestCase
del AbstractGenericWaitTestCase
if __name__ == '__main__':
greentest.main()
This diff is collapsed.
......@@ -20,6 +20,7 @@
# THE SOFTWARE.
import greentest
import greentest.timing
import time
import re
import gevent
......@@ -71,7 +72,7 @@ class TestExceptionInMainloop(greentest.TestCase):
class TestSleep(greentest.GenericWaitTestCase):
class TestSleep(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
gevent.sleep(timeout)
......@@ -80,7 +81,7 @@ class TestSleep(greentest.GenericWaitTestCase):
gevent.sleep(0)
class TestWaiterGet(greentest.GenericWaitTestCase):
class TestWaiterGet(greentest.timing.AbstractGenericWaitTestCase):
def setUp(self):
super(TestWaiterGet, self).setUp()
......
from time import time
import gevent
from gevent import pool
import gevent.pool
from gevent.event import Event
from gevent.queue import Queue
from gevent.timeout import Timeout
import greentest
import greentest.timing
import random
from greentest import ExpectedException
from greentest import six
......@@ -13,12 +14,12 @@ import unittest
class TestCoroutinePool(unittest.TestCase):
klass = pool.Pool
klass = gevent.pool.Pool
def test_apply_async(self):
done = Event()
def some_work(x):
def some_work(_):
done.set()
pool = self.klass(2)
......@@ -125,18 +126,18 @@ class TestCoroutinePool(unittest.TestCase):
pool.join()
def crash(*args, **kw):
def crash(*_args, **_kw):
raise RuntimeError("Whoa")
class FakeFile(object):
def write(*args):
def write(self, *_args):
raise RuntimeError('Whaaa')
class PoolBasicTests(greentest.TestCase):
klass = pool.Pool
klass = gevent.pool.Pool
def test_execute_async(self):
p = self.klass(size=2)
......@@ -208,7 +209,7 @@ class PoolBasicTests(greentest.TestCase):
second = gevent.spawn(gevent.sleep, 1000)
try:
p.add(first)
with self.assertRaises(pool.PoolFull):
with self.assertRaises(gevent.pool.PoolFull):
p.add(second, blocking=False)
finally:
second.kill()
......@@ -223,7 +224,7 @@ class PoolBasicTests(greentest.TestCase):
second = gevent.spawn(gevent.sleep, 1000)
try:
p.add(first)
with self.assertRaises(pool.PoolFull):
with self.assertRaises(gevent.pool.PoolFull):
p.add(second, timeout=0.100)
finally:
second.kill()
......@@ -238,7 +239,7 @@ class PoolBasicTests(greentest.TestCase):
second = gevent.Greenlet(gevent.sleep, 1000)
try:
p.add(first)
with self.assertRaises(pool.PoolFull):
with self.assertRaises(gevent.pool.PoolFull):
p.start(second, timeout=0.100)
finally:
second.kill()
......@@ -304,13 +305,13 @@ if greentest.PYPY and greentest.WIN:
elif greentest.RUNNING_ON_CI or greentest.EXPECT_POOR_TIMER_RESOLUTION:
LARGE_RANGE = 100
class TestPool(greentest.TestCase):
class TestPool(greentest.TestCase): # pylint:disable=too-many-public-methods
__timeout__ = greentest.LARGE_TIMEOUT
size = 1
def setUp(self):
greentest.TestCase.setUp(self)
self.pool = pool.Pool(self.size)
self.pool = gevent.pool.Pool(self.size)
def cleanup(self):
self.pool.join()
......@@ -438,17 +439,11 @@ class TestPool(greentest.TestCase):
running = [0]
def short_running_func(i, j):
def short_running_func(i, _j):
running[0] += 1
return i
# Send two iterables to make sure varargs and kwargs are handled
# correctly
for meth in self.pool.imap_unordered, self.pool.imap:
running[0] = 0
mapping = meth(short_running_func, iterable, iterable,
maxsize=1)
def make_reader(mapping):
# Simulate a long running reader. No matter how many workers
# we have, we will never have a queue more than size 1
def reader():
......@@ -459,7 +454,16 @@ class TestPool(greentest.TestCase):
gevent.sleep(0.01)
self.assertTrue(len(mapping.queue) <= 2, len(mapping.queue))
return result
return reader
# Send two iterables to make sure varargs and kwargs are handled
# correctly
for meth in self.pool.imap_unordered, self.pool.imap:
running[0] = 0
mapping = meth(short_running_func, iterable, iterable,
maxsize=1)
reader = make_reader(mapping)
l = reader()
self.assertEqual(sorted(l), iterable)
......@@ -484,16 +488,16 @@ class TestPool0(greentest.TestCase):
size = 0
def test_wait_full(self):
p = pool.Pool(size=0)
p = gevent.pool.Pool(size=0)
self.assertEqual(0, p.free_count())
self.assertTrue(p.full())
self.assertEqual(0, p.wait_available(timeout=0.01))
class TestJoinSleep(greentest.GenericWaitTestCase):
class TestJoinSleep(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
p = pool.Pool()
p = gevent.pool.Pool()
g = p.spawn(gevent.sleep, 10)
try:
p.join(timeout=timeout)
......@@ -501,10 +505,10 @@ class TestJoinSleep(greentest.GenericWaitTestCase):
g.kill()
class TestJoinSleep_raise_error(greentest.GenericWaitTestCase):
class TestJoinSleep_raise_error(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
p = pool.Pool()
p = gevent.pool.Pool()
g = p.spawn(gevent.sleep, 10)
try:
p.join(timeout=timeout, raise_error=True)
......@@ -516,7 +520,7 @@ class TestJoinEmpty(greentest.TestCase):
switch_expected = False
def test(self):
p = pool.Pool()
p = gevent.pool.Pool()
res = p.join()
self.assertTrue(res, "empty should return true")
......@@ -525,7 +529,7 @@ class TestSpawn(greentest.TestCase):
switch_expected = True
def test(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
self.assertEqual(len(p), 0)
p.spawn(gevent.sleep, 0.1)
self.assertEqual(len(p), 1)
......@@ -535,7 +539,7 @@ class TestSpawn(greentest.TestCase):
self.assertEqual(len(p), 0)
def testSpawnAndWait(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
self.assertEqual(len(p), 0)
p.spawn(gevent.sleep, 0.1)
self.assertEqual(len(p), 1)
......@@ -555,12 +559,12 @@ class TestErrorInIterator(greentest.TestCase):
error_fatal = False
def test(self):
p = pool.Pool(3)
p = gevent.pool.Pool(3)
self.assertRaises(ExpectedException, p.map, lambda x: None, error_iter())
gevent.sleep(0.001)
def test_unordered(self):
p = pool.Pool(3)
p = gevent.pool.Pool(3)
def unordered():
return list(p.imap_unordered(lambda x: None, error_iter()))
......@@ -577,11 +581,11 @@ class TestErrorInHandler(greentest.TestCase):
error_fatal = False
def test_map(self):
p = pool.Pool(3)
p = gevent.pool.Pool(3)
self.assertRaises(ZeroDivisionError, p.map, divide_by, [1, 0, 2])
def test_imap(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
it = p.imap(divide_by, [1, 0, 2])
self.assertEqual(next(it), 1.0)
self.assertRaises(ZeroDivisionError, next, it)
......@@ -589,7 +593,7 @@ class TestErrorInHandler(greentest.TestCase):
self.assertRaises(StopIteration, next, it)
def test_imap_unordered(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
it = p.imap_unordered(divide_by, [1, 0, 2])
self.assertEqual(next(it), 1.0)
self.assertRaises(ZeroDivisionError, next, it)
......
import greentest
from greentest import TestCase, main, GenericGetTestCase
from greentest import TestCase, main
import gevent
from gevent.hub import get_hub, LoopExit
from gevent import util
from gevent import queue
from gevent.queue import Empty, Full
from gevent.event import AsyncResult
from greentest.timing import AbstractGenericGetTestCase
# pylint:disable=too-many-ancestors
class TestQueue(TestCase):
......@@ -372,7 +374,7 @@ class TestJoinEmpty(TestCase):
q.join()
class TestGetInterrupt(GenericGetTestCase):
class TestGetInterrupt(AbstractGenericGetTestCase):
Timeout = Empty
......@@ -397,7 +399,7 @@ class TestGetInterruptChannel(TestGetInterrupt):
kind = queue.Channel
class TestPutInterrupt(GenericGetTestCase):
class TestPutInterrupt(AbstractGenericGetTestCase):
kind = queue.Queue
Timeout = Full
......@@ -430,7 +432,7 @@ class TestPutInterruptChannel(TestPutInterrupt):
return self.kind()
del GenericGetTestCase
del AbstractGenericGetTestCase
if __name__ == '__main__':
......
......@@ -5,86 +5,87 @@ import errno
from gevent import select, socket
import gevent.core
import greentest
import greentest.timing
import unittest
class TestSelect(greentest.GenericWaitTestCase):
class TestSelect(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
select.select([], [], [], timeout)
if sys.platform != 'win32':
class TestSelectRead(greentest.GenericWaitTestCase):
@greentest.skipOnWindows("Cant select on files")
class TestSelectRead(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
r, w = os.pipe()
def wait(self, timeout):
r, w = os.pipe()
try:
select.select([r], [], [], timeout)
finally:
os.close(r)
os.close(w)
# Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606
@unittest.skipIf(sys.platform.startswith('freebsd'),
'skip because of a FreeBSD bug: kern/155606')
def test_errno(self):
# Backported from test_select.py in 3.4
with open(__file__, 'rb') as fp:
fd = fp.fileno()
fp.close()
try:
select.select([r], [], [], timeout)
finally:
os.close(r)
os.close(w)
# Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606
@unittest.skipIf(sys.platform.startswith('freebsd'),
'skip because of a FreeBSD bug: kern/155606')
def test_errno(self):
# Backported from test_select.py in 3.4
with open(__file__, 'rb') as fp:
fd = fp.fileno()
fp.close()
try:
select.select([fd], [], [], 0)
except OSError as err:
# Python 3
self.assertEqual(err.errno, errno.EBADF)
except select.error as err: # pylint:disable=duplicate-except
# Python 2 (select.error is OSError on py3)
self.assertEqual(err.args[0], errno.EBADF)
else:
self.fail("exception not raised")
if hasattr(select, 'poll'):
class TestPollRead(greentest.GenericWaitTestCase):
def wait(self, timeout):
# On darwin, the read pipe is reported as writable
# immediately, for some reason. So we carefully register
# it only for read events (the default is read and write)
r, w = os.pipe()
try:
poll = select.poll()
poll.register(r, select.POLLIN)
poll.poll(timeout * 1000)
finally:
poll.unregister(r)
os.close(r)
os.close(w)
def test_unregister_never_registered(self):
# "Attempting to remove a file descriptor that was
# never registered causes a KeyError exception to be
# raised."
poll = select.poll()
self.assertRaises(KeyError, poll.unregister, 5)
@unittest.skipIf(hasattr(gevent.core, 'libuv'),
"Depending on whether the fileno is reused or not this either crashes or does nothing."
"libuv won't open a watcher for a closed file on linux.")
def test_poll_invalid(self):
with open(__file__, 'rb') as fp:
fd = fp.fileno()
poll = select.poll()
poll.register(fd, select.POLLIN)
# Close after registering; libuv refuses to even
# create a watcher if it would get EBADF (so this turns into
# a test of whether or not we successfully initted the watcher).
fp.close()
result = poll.poll(0)
self.assertEqual(result, [(fd, select.POLLNVAL)]) # pylint:disable=no-member
select.select([fd], [], [], 0)
except OSError as err:
# Python 3
self.assertEqual(err.errno, errno.EBADF)
except select.error as err: # pylint:disable=duplicate-except
# Python 2 (select.error is OSError on py3)
self.assertEqual(err.args[0], errno.EBADF)
else:
self.fail("exception not raised")
@unittest.skipUnless(hasattr(select, 'poll'), "Needs poll")
@greentest.skipOnWindows("Cant poll on files")
class TestPollRead(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
# On darwin, the read pipe is reported as writable
# immediately, for some reason. So we carefully register
# it only for read events (the default is read and write)
r, w = os.pipe()
try:
poll = select.poll()
poll.register(r, select.POLLIN)
poll.poll(timeout * 1000)
finally:
poll.unregister(r)
os.close(r)
os.close(w)
def test_unregister_never_registered(self):
# "Attempting to remove a file descriptor that was
# never registered causes a KeyError exception to be
# raised."
poll = select.poll()
self.assertRaises(KeyError, poll.unregister, 5)
@unittest.skipIf(hasattr(gevent.core, 'libuv'),
"Depending on whether the fileno is reused or not this either crashes or does nothing."
"libuv won't open a watcher for a closed file on linux.")
def test_poll_invalid(self):
with open(__file__, 'rb') as fp:
fd = fp.fileno()
poll = select.poll()
poll.register(fd, select.POLLIN)
# Close after registering; libuv refuses to even
# create a watcher if it would get EBADF (so this turns into
# a test of whether or not we successfully initted the watcher).
fp.close()
result = poll.poll(0)
self.assertEqual(result, [(fd, select.POLLNVAL)]) # pylint:disable=no-member
class TestSelectTypes(greentest.TestCase):
......
......@@ -9,6 +9,7 @@ import unittest
import greentest
from functools import wraps
from greentest import six
from greentest import LARGE_TIMEOUT
# we use threading on purpose so that we can test both regular and gevent sockets with the same code
from threading import Thread as _Thread
......@@ -42,7 +43,7 @@ class TestTCP(greentest.TestCase):
__timeout__ = None
TIMEOUT_ERROR = socket.timeout
long_data = ", ".join([str(x) for x in range(20000)])
if six.PY3:
if not isinstance(long_data, bytes):
long_data = long_data.encode('ascii')
def setUp(self):
......@@ -341,7 +342,7 @@ def get_port():
class TestCreateConnection(greentest.TestCase):
__timeout__ = 5
__timeout__ = LARGE_TIMEOUT
def test_refuses(self):
with self.assertRaises(socket.error) as cm:
......
......@@ -25,7 +25,9 @@ if getattr(resolver, 'pool', None) is not None:
from greentest.sysinfo import RESOLVER_NOT_SYSTEM
from greentest.sysinfo import RESOLVER_DNSPYTHON
from greentest.sysinfo import RESOLVER_ARES
from greentest.sysinfo import PY2
import greentest.timing
assert gevent_socket.gaierror is socket.gaierror
......@@ -444,6 +446,8 @@ class SanitizedHostsFile(HostsFile):
continue
yield name, addr
@greentest.skipIf(greentest.RUNNING_ON_TRAVIS and RESOLVER_ARES,
"This sometimes randomly fails on Travis with ares, beginning Feb 13, 2018")
class TestEtcHosts(TestCase):
MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
......@@ -567,7 +571,7 @@ add(TestInternational, u'президент.рф', 'russian',
add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
class TestInterrupted_gethostbyname(greentest.timing.AbstractGenericWaitTestCase):
# There are refs to a Waiter in the C code that don't go
# away yet; one gc may or may not do it.
......
......@@ -123,3 +123,5 @@ test_timeout.py
# implicitly for localhost, which is covered well enough
# elsewhere that we don't need to spend the 20s (*2)
test_asyncore.py
test___config.py
......@@ -36,7 +36,7 @@ commands =
basepython =
python2.7
setenv =
GEVENT_CORE_CFFI_ONLY=1
GEVENT_LOOP=libev-cffi
commands =
make basictest
......@@ -44,7 +44,7 @@ commands =
basepython =
python2.7
setenv =
GEVENT_CORE_CFFI_ONLY=libuv
GEVENT_LOOP=libuv-cffi
commands =
make basictest
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment