Skip to content

G
gevent
Commit 0a0c16fb authored by Jason Madden's avatar Jason Madden Committed by GitHub
Browse files
Options

Merge pull request #1096 from gevent/config-issue1090 

Centralize configuration in a ``gevent.config`` object.
parents 654e8302 96b35730
Hide whitespace changes
Inline Side-by-side
Showing with 1189 additions and 588 deletions
CHANGES.rst
View file @ 0a0c16fb
......@@ -46,6 +46,18 @@
:mod:`time`, much like :mod:`gevent.socket` can be imported instead
of :mod:`socket`.
- Centralize all gevent configuration in an object at
``gevent.config``, allowing for gevent to be configured through code
and not *necessarily* environment variables, and also provide a
centralized place for documentation. See :issue:`1090`.
- The new ``GEVENT_CORE_CFFI_ONLY`` environment variable has been
replaced with the pre-existing ``GEVENT_LOOP`` environment
variable. That variable may take the values ``libev-cext``,
``libev-cffi``, or ``libuv-cffi``, (or be a list in preference
order, or be a dotted name; it may also be assigned to an
object in Python code at ``gevent.config.loop``).
1.3a1 (2018-01-27)
==================
......
Makefile
View file @ 0a0c16fb
......@@ -45,46 +45,43 @@ prospector:
lint: prospector
test_prelim:
which ${PYTHON}
${PYTHON} --version
${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
make bench
@which ${PYTHON}
@${PYTHON} --version
@${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
@${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
@${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
@make bench
# Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44
# echo -e "travis_fold:start:${GEVENT_CORE_CFFI_ONLY}\033[33;1m${GEVENT_CORE_CFFI_ONLY}\033[0m"
# Make calls /bin/echo, which doesn't support the -e option, which is part of the bash builtin.
# we need a python script to do this, or possible the GNU make shell function
basictest: test_prelim
${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
@${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
cd src/greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config known_failures.py --quiet
${PYTHON} scripts/travis.py fold_end basictest
@${PYTHON} scripts/travis.py fold_end basictest
alltest: basictest
${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
@${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
${PYTHON} scripts/travis.py fold_end ares
${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
@${PYTHON} scripts/travis.py fold_end ares
@${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
${PYTHON} scripts/travis.py fold_end dnspython
@${PYTHON} scripts/travis.py fold_end dnspython
# In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
# text. The monkey-patched stdlib tests were specifically included here.
# However, we now always also test on AppVeyor (Windows) which only has GEVENT_FILE=thread,
# so we can save a lot of CI time by reducing the set and excluding the stdlib tests without
# losing any coverage. See the `threadfiletest` for what command used to run.
${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
@${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py test__*subprocess*.py --quiet
${PYTHON} scripts/travis.py fold_end thread
@${PYTHON} scripts/travis.py fold_end thread
threadfiletest:
cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py `grep -l subprocess test_*.py` --quiet
allbackendtest:
${PYTHON} scripts/travis.py fold_start default "Testing default backend"
GEVENT_CORE_CFFI_ONLY= GEVENTTEST_COVERAGE=1 make alltest
${PYTHON} scripts/travis.py fold_end default
@${PYTHON} scripts/travis.py fold_start default "Testing default backend"
GEVENTTEST_COVERAGE=1 make alltest
@${PYTHON} scripts/travis.py fold_end default
GEVENTTEST_COVERAGE=1 make cffibackendtest
# because we set parallel=true, each run produces new and different coverage files; they all need
# to be combined
......@@ -92,17 +89,17 @@ allbackendtest:
cffibackendtest:
${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
GEVENT_CORE_CFFI_ONLY=libuv GEVENTTEST_COVERAGE=1 make alltest
${PYTHON} scripts/travis.py fold_end libuv
${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
GEVENT_CORE_CFFI_ONLY=libev make alltest
${PYTHON} scripts/travis.py fold_end libev
@${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
@${PYTHON} scripts/travis.py fold_end libuv
@${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
GEVENT_LOOP=libev-cffi make alltest
@${PYTHON} scripts/travis.py fold_end libev
leaktest: test_prelim
${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
@${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
cd src/greentest && GEVENT_RESOLVER=thread GEVENTTEST_LEAKCHECK=1 ${PYTHON} testrunner.py --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
${PYTHON} scripts/travis.py fold_end leaktest
@${PYTHON} scripts/travis.py fold_end leaktest
bench:
${PYTHON} src/greentest/bench_sendall.py
......@@ -166,8 +163,8 @@ $(PYPY3):
develop:
${PYTHON} scripts/travis.py fold_start install "Installing gevent"
echo python is at `which $(PYTHON)`
@${PYTHON} scripts/travis.py fold_start install "Installing gevent"
@echo python is at `which $(PYTHON)`
# First install a newer pip so that it can use the wheel cache
# (only needed until travis upgrades pip to 7.x; note that the 3.5
# environment uses pip 7.1 by default)
......@@ -176,7 +173,7 @@ develop:
# disables the cache.
# We need wheel>=0.26 on Python 3.5. See previous revisions.
GEVENTSETUP_EV_VERIFY=3 ${PYTHON} -m pip install -U -r dev-requirements.txt
${PYTHON} scripts/travis.py fold_end install
@${PYTHON} scripts/travis.py fold_end install
test-py27: $(PY27)
PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest allbackendtest
......@@ -200,7 +197,13 @@ test-pypy3: $(PYPY3)
PYTHON=$(PYPY3) PATH=$(BUILD_RUNTIMES)/versions/pypy3.5_5101/bin:$(PATH) make develop basictest
test-py27-noembed: $(PY27)
@python2.7.14 scripts/travis.py fold_start conf_libev "Configuring libev"
cd deps/libev && ./configure --disable-dependency-tracking && make
@python2.7.14 scripts/travis.py fold_end conf_libev
@python2.7.14 scripts/travis.py fold_start conf_cares "Configuring cares"
cd deps/c-ares && ./configure --disable-dependency-tracking && make
@python2.7.14 scripts/travis.py fold_end conf_cares
@python2.7.14 scripts/travis.py fold_start conf_libuv "Configuring libuv"
cd deps/libuv && ./autogen.sh && ./configure --disable-static && make
CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
@python2.7.14 scripts/travis.py fold_end conf_libuv
CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_LOOP=libev-cext PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
appveyor.yml
View file @ 0a0c16fb
......@@ -26,7 +26,7 @@ environment:
PYTHON_VERSION: "3.6.x" # currently 3.6.0
PYTHON_ARCH: "64"
PYTHON_EXE: python
GEVENT_CORE_CFFI_ONLY: libuv
GEVENT_LOOP: libuv
- PYTHON: "C:\\Python27-x64"
PYTHON_VERSION: "2.7.x" # currently 2.7.13
......
doc/conf.py
View file @ 0a0c16fb
......@@ -87,7 +87,7 @@ release = version
exclude_trees = ['_build']
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
default_role = 'obj'
# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = True
......
doc/gevent.rst
View file @ 0a0c16fb
......@@ -191,3 +191,9 @@ Timeouts
:special-members: __enter__, __exit__
.. autofunction:: with_timeout
Configuration
=============
.. autoclass:: gevent._config.Config
src/gevent/__init__.py
View file @ 0a0c16fb
......@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet
to provide a high-level synchronous API on top of libev event loop.
See http://www.gevent.org/ for the documentation.
.. versionchanged:: 1.3a2
Add the `config` object.
"""
from __future__ import absolute_import
......@@ -43,6 +46,8 @@ __all__ = [
'reinit',
'getswitchinterval',
'setswitchinterval',
# Added in 1.3a2
'config',
]
......@@ -72,11 +77,14 @@ except AttributeError:
global _switchinterval
_switchinterval = interval
from gevent._config import config
from gevent.hub import get_hub, iwait, wait
from gevent.greenlet import Greenlet, joinall, killall
joinall = joinall # export for pylint
spawn = Greenlet.spawn
spawn_later = Greenlet.spawn_later
#: The singleton configuration object for gevent.
config = config
from gevent.timeout import Timeout, with_timeout
from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit
......
src/gevent/_config.py 0 → 100644
View file @ 0a0c16fb
# Copyright (c) 2018 gevent. See LICENSE for details.
"""
gevent tunables.
This should be used as ``from gevent import config``. That variable
is an object of :class:`Config`.
.. versionadded:: 1.3a2
"""
from __future__ import print_function, absolute_import, division
import importlib
import os
import sys
import textwrap
from gevent._compat import string_types
__all__ = [
'config',
]
ALL_SETTINGS = []
class SettingType(type):
# pylint:disable=bad-mcs-classmethod-argument
def __new__(cls, name, bases, cls_dict):
if name == 'Setting':
return type.__new__(cls, name, bases, cls_dict)
cls_dict["order"] = len(ALL_SETTINGS)
if 'name' not in cls_dict:
cls_dict['name'] = name.lower()
if 'environment_key' not in cls_dict:
cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()
new_class = type.__new__(cls, name, bases, cls_dict)
new_class.fmt_desc(cls_dict.get("desc", ""))
new_class.__doc__ = new_class.desc
ALL_SETTINGS.append(new_class)
if new_class.document:
setting_name = cls_dict['name']
def getter(self):
return self.settings[setting_name].get()
def setter(self, value): # pragma: no cover
# The setter should never be hit, Config has a
# __setattr__ that would override. But for the sake
# of consistency we provide one.
self.settings[setting_name].set(value)
prop = property(getter, setter, doc=new_class.__doc__)
setattr(Config, cls_dict['name'], prop)
return new_class
def fmt_desc(cls, desc):
desc = textwrap.dedent(desc).strip()
if hasattr(cls, 'shortname_map'):
desc += (
"\n\nThis is an importable value. It can be "
"given as a string naming an importable object, "
"or a list of strings in preference order and the first "
"successfully importable object will be used. (Separate values "
"in the environment variable with commas.) "
"It can also be given as the callable object itself (in code). "
)
if cls.shortname_map:
desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
if getattr(cls.validate, '__doc__'):
desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
default = "`%s`" % (cls.default,)
else:
default = "`%r`" % (cls.default,)
desc += "\n\nThe default value is %s" % (default,)
desc += ("\n\nThe environment variable ``%s`` "
"can be used to control this." % (cls.environment_key,))
setattr(cls, "desc", desc)
return desc
def validate_invalid(value):
raise ValueError("Not a valid value: %r" % (value,))
def validate_bool(value):
"""
This is a boolean value.
In the environment variable, it may be given as ``1``, ``true``,
``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
``no`` for `False`.
"""
if isinstance(value, string_types):
value = value.lower().strip()
if value in ('1', 'true', 'on', 'yes'):
value = True
elif value in ('0', 'false', 'off', 'no') or not value:
value = False
else:
raise ValueError("Invalid boolean string: %r" % (value,))
return bool(value)
def validate_anything(value):
return value
class Setting(object):
name = None
value = None
validate = staticmethod(validate_invalid)
default = None
environment_key = None
document = True
desc = """\
A long ReST description.
The first line should be a single sentence.
"""
def _convert(self, value):
if isinstance(value, string_types):
return value.split(',')
return value
def _default(self):
result = os.environ.get(self.environment_key, self.default)
result = self._convert(result)
return result
def get(self):
# If we've been specifically set, return it
if 'value' in self.__dict__:
return self.value
# Otherwise, read from the environment and reify
# so we return consistent results.
self.value = self.validate(self._default())
return self.value
def set(self, val):
self.value = self.validate(self._convert(val))
Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))
def make_settings():
"""
Return fresh instances of all classes defined in `ALL_SETTINGS`.
"""
settings = {}
for setting_kind in ALL_SETTINGS:
setting = setting_kind()
assert setting.name not in settings
settings[setting.name] = setting
return settings
class Config(object):
"""
Global configuration for gevent.
There is one instance of this object at ``gevent.config``. If you
are going to make changes in code, instead of using the documented
environment variables, you need to make the changes before using
any parts of gevent that might need those settings. For example::
>>> from gevent import config
>>> config.fileobject = 'thread'
>>> from gevent import fileobject
>>> fileobject.FileObject.__name__
'FileObjectThread'
.. versionadded:: 1.3a2
"""
def __init__(self):
self.settings = make_settings()
def __getattr__(self, name):
if name not in self.settings:
raise AttributeError("No configuration setting for: %r" % name)
return self.settings[name].get()
def __setattr__(self, name, value):
if name != "settings" and name in self.settings:
self.set(name, value)
else:
super(Config, self).__setattr__(name, value)
def set(self, name, value):
if name not in self.settings:
raise AttributeError("No configuration setting for: %r" % name)
self.settings[name].set(value)
def __dir__(self):
return list(self.settings)
class ImportableSetting(object):
def _import(self, path, _NONE=object):
# pylint:disable=too-many-branches
if isinstance(path, list):
if not path:
raise ImportError('Cannot import from empty list: %r' % (path, ))
for item in path[:-1]:
try:
return self._import(item)
except ImportError:
pass
return self._import(path[-1])
if not isinstance(path, string_types):
return path
if '.' not in path:
raise ImportError("Cannot import %r. "
"Required format: [path/][package.]module.class. "
"Or choice from %r"
% (path, list(self.shortname_map)))
if '/' in path:
# This is dangerous, subject to race conditions, and
# may not work properly for things like namespace packages
import warnings
warnings.warn("Absolute paths are deprecated and will be removed in 1.4."
"Please put the package on sys.path first",
DeprecationWarning)
package_path, path = path.rsplit('/', 1)
sys.path = [package_path] + sys.path
else:
package_path = None
try:
module, item = path.rsplit('.', 1)
module = importlib.import_module(module)
x = getattr(module, item, _NONE)
if x is _NONE:
raise ImportError('Cannot import %r from %r' % (item, module))
return x
finally:
if package_path:
try:
sys.path.remove(package_path)
except ValueError: # pragma: no cover
pass
shortname_map = {}
def validate(self, value):
if isinstance(value, type):
return value
return self._import([self.shortname_map.get(x, x) for x in value])
class Resolver(ImportableSetting, Setting):
desc = """\
The callable that will be used to create
:attr:`gevent.hub.Hub.resolver`.
See :doc:`dns` for more information.
"""
default = [
'thread',
'dnspython',
'ares',
'block',
]
shortname_map = {
'ares': 'gevent.resolver.ares.Resolver',
'thread': 'gevent.resolver.thread.Resolver',
'block': 'gevent.resolver.blocking.Resolver',
'dnspython': 'gevent.resolver.dnspython.Resolver',
}
class Threadpool(ImportableSetting, Setting):
desc = """\
The kind of threadpool we use.
"""
default = 'gevent.threadpool.ThreadPool'
class Loop(ImportableSetting, Setting):
desc = """\
The kind of the loop we use.
"""
default = [
'libev-cext',
'libev-cffi',
'libuv-cffi',
]
shortname_map = {
'libev-cext': 'gevent.libev.corecext.loop',
'libev-cffi': 'gevent.libev.corecffi.loop',
'libuv-cffi': 'gevent.libuv.loop.loop',
}
shortname_map['libuv'] = shortname_map['libuv-cffi']
class FormatContext(ImportableSetting, Setting):
name = 'format_context'
# using pprint.pformat can override custom __repr__ methods on dict/list
# subclasses, which can be a security concern
default = 'pprint.saferepr'
class LibevBackend(Setting):
name = 'libev_backend'
environment_key = 'GEVENT_BACKEND'
desc = """\
The backend for libev, such as 'select'
"""
default = None
validate = staticmethod(validate_anything)
class FileObject(ImportableSetting, Setting):
desc = """\
The kind of ``FileObject`` we will use.
See :mod:`gevent.fileobject` for a detailed description.
"""
environment_key = 'GEVENT_FILE'
default = [
'posix',
'thread',
]
shortname_map = {
'thread': 'gevent._fileobjectcommon.FileObjectThread',
'posix': 'gevent._fileobjectposix.FileObjectPosix',
'block': 'gevent._fileobjectcommon.FileObjectBlock'
}
class WatchChildren(Setting):
desc = """\
Should we *not* watch children with the event loop watchers?
This is an advanced setting.
See :mod:`gevent.os` for a detailed description.
"""
name = 'disable_watch_children'
environment_key = 'GEVENT_NOWAITPID'
default = False
validate = staticmethod(validate_bool)
class TraceMalloc(Setting):
name = 'trace_malloc'
environment_key = 'PYTHONTRACEMALLOC'
default = False
validate = staticmethod(validate_bool)
desc = """\
Should FFI objects track their allocation?
This is only useful for low-level debugging.
On Python 3, this environment variable is built in to the
interpreter, and it may also be set with the ``-X
tracemalloc`` command line argument.
On Python 2, gevent interprets this argument and adds extra
tracking information for FFI objects.
"""
# The ares settings are all interpreted by
# gevent/resolver/ares.pyx, so we don't do
# any validation here.
class AresSettingMixin(object):
document = False
@property
def kwarg_name(self):
return self.name[5:]
validate = staticmethod(validate_anything)
_convert = staticmethod(validate_anything)
class AresFlags(AresSettingMixin, Setting):
name = 'ares_flags'
default = None
environment_key = 'GEVENTARES_FLAGS'
class AresTimeout(AresSettingMixin, Setting):
name = 'ares_timeout'
default = None
environment_key = 'GEVENTARES_TIMEOUT'
class AresTries(AresSettingMixin, Setting):
name = 'ares_tries'
default = None
environment_key = 'GEVENTARES_TRIES'
class AresNdots(AresSettingMixin, Setting):
name = 'ares_ndots'
default = None
environment_key = 'GEVENTARES_NDOTS'
class AresUDPPort(AresSettingMixin, Setting):
name = 'ares_udp_port'
default = None
environment_key = 'GEVENTARES_UDP_PORT'
class AresTCPPort(AresSettingMixin, Setting):
name = 'ares_tcp_port'
default = None
environment_key = 'GEVENTARES_TCP_PORT'
class AresServers(AresSettingMixin, Setting):
document = True
name = 'ares_servers'
default = None
environment_key = 'GEVENTARES_SERVERS'
config = Config()
# Go ahead and attempt to import the loop when this class is
# instantiated. The hub won't work if the loop can't be found. This
# can solve problems with the class being imported from multiple
# threads at once, leading to one of the imports failing.
# factories are themselves handled lazily. See #687.
# Don't cache it though, in case the user re-configures through the
# API.
try:
Loop().get()
except ImportError: # pragma: no cover
pass
src/gevent/_ffi/watcher.py
View file @ 0a0c16fb
......@@ -10,14 +10,17 @@ import signal as signalmodule
import functools
import warnings
from gevent._config import config
try:
from tracemalloc import get_object_traceback
def tracemalloc(init):
# PYTHONTRACEMALLOC env var controls this.
# PYTHONTRACEMALLOC env var controls this on Python 3.
return init
except ImportError: # Python < 3.4
if os.getenv('PYTHONTRACEMALLOC'):
if config.trace_malloc:
# Use the same env var to turn this on for Python 2
import traceback
......
src/gevent/_fileobjectcommon.py
View file @ 0a0c16fb
from __future__ import absolute_import, print_function, division
try:
from errno import EBADF
except ImportError:
EBADF = 9
import os
from io import TextIOWrapper
import functools
import sys
from gevent.hub import get_hub
from gevent._compat import integer_types
from gevent._compat import reraise
from gevent.lock import Semaphore, DummySemaphore
class cancel_wait_ex(IOError):
......@@ -136,3 +146,130 @@ class FileObjectBase(object):
def __exit__(self, *args):
self.close()
class FileObjectBlock(FileObjectBase):
def __init__(self, fobj, *args, **kwargs):
closefd = kwargs.pop('close', True)
if kwargs:
raise TypeError('Unexpected arguments: %r' % kwargs.keys())
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectBlock does not support close=False on an fd.')
fobj = os.fdopen(fobj, *args)
super(FileObjectBlock, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
fobj.close()
class FileObjectThread(FileObjectBase):
"""
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
.. caution::
Attempting to change the threadpool or lock of an existing FileObjectThread
has undefined consequences.
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
"""
:param fobj: The underlying file-like object to wrap, or an integer fileno
that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
in any particular order, only that no two operations will be attempted at the
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
file operations with an external resource.
:keyword bool close: If True (the default) then when this object is closed,
the underlying object is closed as well.
"""
closefd = close
self.threadpool = threadpool or get_hub().threadpool
self.lock = lock
if self.lock is True:
self.lock = Semaphore()
elif not self.lock:
self.lock = DummySemaphore()
if not hasattr(self.lock, '__enter__'):
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectThread does not support close=False on an fd.')
if mode is None:
assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
fobj = os.fdopen(fobj)
else:
fobj = os.fdopen(fobj, mode, bufsize)
self.__io_holder = [fobj] # signal for _wrap_method
super(FileObjectThread, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
self.__io_holder[0] = None # for _wrap_method
try:
with self.lock:
self.threadpool.apply(fobj.flush)
finally:
if closefd:
# Note that we're not taking the lock; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged by the pool
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _do_delegate_methods(self):
super(FileObjectThread, self)._do_delegate_methods()
if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
self.read1 = self.read
self.__io_holder[0] = self._io
def _extra_repr(self):
return ' threadpool=%r' % (self.threadpool,)
def __iter__(self):
return self
def next(self):
line = self.readline()
if line:
return line
raise StopIteration
__next__ = next
def _wrap_method(self, method):
# NOTE: We are careful to avoid introducing a refcycle
# within self. Our wrapper cannot refer to self.
io_holder = self.__io_holder
lock = self.lock
threadpool = self.threadpool
@functools.wraps(method)
def thread_method(*args, **kwargs):
if io_holder[0] is None:
# This is different than FileObjectPosix, etc,
# because we want to save the expensive trip through
# the threadpool.
raise FileObjectClosed()
with lock:
return threadpool.apply(method, args, kwargs)
return thread_method
src/gevent/core.py
View file @ 0a0c16fb
# Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
"""
Deprecated; this does not reflect all the possible options
and its interface varies.
.. versionchanged:: 1.3a2
Deprecated.
"""
from __future__ import absolute_import
import os
import sys
from gevent._config import config
from gevent._util import copy_globals
try:
if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
raise ImportError("Not attempting corecext")
from gevent.libev import corecext as _core
except ImportError:
if os.environ.get('GEVENT_CORE_CEXT_ONLY'):
raise
# CFFI/PyPy
lib = os.environ.get('GEVENT_CORE_CFFI_ONLY')
if lib == 'libuv':
from gevent.libuv import loop as _core
else:
try:
from gevent.libev import corecffi as _core
except ImportError:
from gevent.libuv import loop as _core
_core = sys.modules[config.loop.__module__]
copy_globals(_core, globals())
......
src/gevent/fileobject.py
View file @ 0a0c16fb
......@@ -35,31 +35,12 @@ Classes
"""
from __future__ import absolute_import
import functools
import sys
import os
from gevent._fileobjectcommon import FileObjectClosed
from gevent._fileobjectcommon import FileObjectBase
from gevent.hub import get_hub
from gevent._compat import integer_types
from gevent._compat import reraise
from gevent.lock import Semaphore, DummySemaphore
PYPY = hasattr(sys, 'pypy_version_info')
if hasattr(sys, 'exc_clear'):
def _exc_clear():
sys.exc_clear()
else:
def _exc_clear():
return
from gevent._config import config
__all__ = [
'FileObjectPosix',
'FileObjectThread',
'FileObjectBlock',
'FileObject',
]
......@@ -71,149 +52,10 @@ else:
del fcntl
from gevent._fileobjectposix import FileObjectPosix
from gevent._fileobjectcommon import FileObjectThread
from gevent._fileobjectcommon import FileObjectBlock
class FileObjectThread(FileObjectBase):
"""
A file-like object wrapping another file-like object, performing all blocking
operations on that object in a background thread.
.. caution::
Attempting to change the threadpool or lock of an existing FileObjectThread
has undefined consequences.
.. versionchanged:: 1.1b1
The file object is closed using the threadpool. Note that whether or
not this action is synchronous or asynchronous is not documented.
"""
def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
"""
:param fobj: The underlying file-like object to wrap, or an integer fileno
that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
:keyword bool lock: If True (the default) then all operations will
be performed one-by-one. Note that this does not guarantee that, if using
this file object from multiple threads/greenlets, operations will be performed
in any particular order, only that no two operations will be attempted at the
same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
file operations with an external resource.
:keyword bool close: If True (the default) then when this object is closed,
the underlying object is closed as well.
"""
closefd = close
self.threadpool = threadpool or get_hub().threadpool
self.lock = lock
if self.lock is True:
self.lock = Semaphore()
elif not self.lock:
self.lock = DummySemaphore()
if not hasattr(self.lock, '__enter__'):
raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectThread does not support close=False on an fd.')
if mode is None:
assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
fobj = os.fdopen(fobj)
else:
fobj = os.fdopen(fobj, mode, bufsize)
self.__io_holder = [fobj] # signal for _wrap_method
super(FileObjectThread, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
self.__io_holder[0] = None # for _wrap_method
try:
with self.lock:
self.threadpool.apply(fobj.flush)
finally:
if closefd:
# Note that we're not taking the lock; older code
# did fobj.close() without going through the threadpool at all,
# so acquiring the lock could potentially introduce deadlocks
# that weren't present before. Avoiding the lock doesn't make
# the existing race condition any worse.
# We wrap the close in an exception handler and re-raise directly
# to avoid the (common, expected) IOError from being logged by the pool
def close():
try:
fobj.close()
except: # pylint:disable=bare-except
return sys.exc_info()
exc_info = self.threadpool.apply(close)
if exc_info:
reraise(*exc_info)
def _do_delegate_methods(self):
super(FileObjectThread, self)._do_delegate_methods()
if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
self.read1 = self.read
self.__io_holder[0] = self._io
def _extra_repr(self):
return ' threadpool=%r' % (self.threadpool,)
def __iter__(self):
return self
def next(self):
line = self.readline()
if line:
return line
raise StopIteration
__next__ = next
def _wrap_method(self, method):
# NOTE: We are careful to avoid introducing a refcycle
# within self. Our wrapper cannot refer to self.
io_holder = self.__io_holder
lock = self.lock
threadpool = self.threadpool
@functools.wraps(method)
def thread_method(*args, **kwargs):
if io_holder[0] is None:
# This is different than FileObjectPosix, etc,
# because we want to save the expensive trip through
# the threadpool.
raise FileObjectClosed()
with lock:
return threadpool.apply(method, args, kwargs)
return thread_method
try:
FileObject = FileObjectPosix
except NameError:
FileObject = FileObjectThread
class FileObjectBlock(FileObjectBase):
def __init__(self, fobj, *args, **kwargs):
closefd = kwargs.pop('close', True)
if kwargs:
raise TypeError('Unexpected arguments: %r' % kwargs.keys())
if isinstance(fobj, integer_types):
if not closefd:
# we cannot do this, since fdopen object will close the descriptor
raise TypeError('FileObjectBlock does not support close=False on an fd.')
fobj = os.fdopen(fobj, *args)
super(FileObjectBlock, self).__init__(fobj, closefd)
def _do_close(self, fobj, closefd):
fobj.close()
config = os.environ.get('GEVENT_FILE')
if config:
klass = {'thread': 'gevent.fileobject.FileObjectThread',
'posix': 'gevent.fileobject.FileObjectPosix',
'block': 'gevent.fileobject.FileObjectBlock'}.get(config, config)
if klass.startswith('gevent.fileobject.'):
FileObject = globals()[klass.split('.', 2)[-1]]
else:
from gevent.hub import _import
FileObject = _import(klass)
del klass
# None of the possible objects can live in this module because
# we would get an import cycle and the config couldn't be set from code.
FileObject = config.fileobject
src/gevent/hub.py
View file @ 0a0c16fb
......@@ -26,6 +26,7 @@ __all__ = [
'Waiter',
]
from gevent._config import config
from gevent._compat import string_types
from gevent._compat import xrange
from gevent._util import _NONE
......@@ -389,89 +390,29 @@ def set_hub(hub):
_threadlocal.hub = hub
def _import(path):
# pylint:disable=too-many-branches
if isinstance(path, list):
if not path:
raise ImportError('Cannot import from empty list: %r' % (path, ))
for item in path[:-1]:
try:
return _import(item)
except ImportError:
pass
return _import(path[-1])
if not isinstance(path, string_types):
return path
if '.' not in path:
raise ImportError("Cannot import %r (required format: [path/][package.]module.class)" % path)
if '/' in path:
# This is dangerous, subject to race conditions, and
# may not work properly for things like namespace packages
import warnings
warnings.warn("Absolute paths are deprecated. Please put the package "
"on sys.path first",
DeprecationWarning)
package_path, path = path.rsplit('/', 1)
sys.path = [package_path] + sys.path
else:
package_path = None
try:
module, item = path.rsplit('.', 1)
x = __import__(module)
for attr in path.split('.')[1:]:
oldx = x
x = getattr(x, attr, _NONE)
if x is _NONE:
raise ImportError('Cannot import %r from %r' % (attr, oldx))
return x
finally:
if '/' in path:
try:
sys.path.remove(package_path)
except ValueError:
pass
def config(default, envvar):
def _config(default, envvar):
result = os.environ.get(envvar) or default # absolute import gets confused pylint: disable=no-member
if isinstance(result, string_types):
return result.split(',')
return result
def resolver_config(default, envvar):
result = config(default, envvar)
return [_resolvers.get(x, x) for x in result]
_resolvers = {
'ares': 'gevent.resolver.ares.Resolver',
'thread': 'gevent.resolver.thread.Resolver',
'block': 'gevent.resolver.blocking.Resolver',
'dnspython': 'gevent.resolver.dnspython.Resolver',
}
_DEFAULT_LOOP_CLASS = 'gevent.core.loop'
class Hub(RawGreenlet):
"""A greenlet that runs the event loop.
"""
A greenlet that runs the event loop.
It is created automatically by :func:`get_hub`.
**Switching**
.. rubric:: Switching
Every time this greenlet (i.e., the event loop) is switched *to*, if
the current greenlet has a ``switch_out`` method, it will be called. This
allows a greenlet to take some cleanup actions before yielding control. This method
should not call any gevent blocking functions.
Every time this greenlet (i.e., the event loop) is switched *to*,
if the current greenlet has a ``switch_out`` method, it will be
called. This allows a greenlet to take some cleanup actions before
yielding control. This method should not call any gevent blocking
functions.
"""
#: If instances of these classes are raised into the event loop,
......@@ -483,37 +424,8 @@ class Hub(RawGreenlet):
#: do not get logged/printed when raised by the event loop.
NOT_ERROR = (GreenletExit, SystemExit)
loop_class = config(_DEFAULT_LOOP_CLASS, 'GEVENT_LOOP')
# For the standard class, go ahead and import it when this class
# is defined. This is no loss of generality because the envvar is
# only read when this class is defined, and we know that the
# standard class will be available. This can solve problems with
# the class being imported from multiple threads at once, leading
# to one of the imports failing. Only do this for the object we
# need in the constructor, as the rest of the factories are
# themselves handled lazily. See #687. (People using a custom loop_class
# can probably manage to get_hub() from the main thread or otherwise import
# that loop_class themselves.)
if loop_class == [_DEFAULT_LOOP_CLASS]:
loop_class = [_import(loop_class)]
resolver_class = [
'gevent.resolver.thread.Resolver',
'gevent.resolver.dnspython.Resolver',
'gevent.resolver.ares.Resolver',
'gevent.resolver.blacking.Resolver',
]
#: The class or callable object, or the name of a factory function or class,
#: that will be used to create :attr:`resolver`. By default, configured according to
#: :doc:`dns`. If a list, a list of objects in preference order.
resolver_class = resolver_config(resolver_class, 'GEVENT_RESOLVER')
threadpool_class = config('gevent.threadpool.ThreadPool', 'GEVENT_THREADPOOL')
backend = config(None, 'GEVENT_BACKEND')
threadpool_size = 10
# using pprint.pformat can override custom __repr__ methods on dict/list
# subclasses, which can be a security concern
format_context = 'pprint.saferepr'
threadpool_size = 10
def __init__(self, loop=None, default=None):
......@@ -530,13 +442,21 @@ class Hub(RawGreenlet):
else:
if default is None and get_ident() != MAIN_THREAD:
default = False
loop_class = _import(self.loop_class)
if loop is None:
loop = self.backend
self.loop = loop_class(flags=loop, default=default)
self.loop = self.loop_class(flags=loop, default=default) # pylint:disable=not-callable
self._resolver = None
self._threadpool = None
self.format_context = _import(self.format_context)
self.format_context = config.format_context
@property
def loop_class(self):
return config.loop
@property
def backend(self):
return config.libev_backend
def __repr__(self):
if self.loop is None:
......@@ -787,11 +707,13 @@ class Hub(RawGreenlet):
if _threadlocal.hub is self:
_threadlocal.hub = None
@property
def resolver_class(self):
return config.resolver
def _get_resolver(self):
if self._resolver is None:
if self.resolver_class is not None:
self.resolver_class = _import(self.resolver_class)
self._resolver = self.resolver_class(hub=self)
self._resolver = self.resolver_class(hub=self) # pylint:disable=not-callable
return self._resolver
def _set_resolver(self, value):
......@@ -802,11 +724,15 @@ class Hub(RawGreenlet):
resolver = property(_get_resolver, _set_resolver, _del_resolver)
@property
def threadpool_class(self):
return config.threadpool
def _get_threadpool(self):
if self._threadpool is None:
if self.threadpool_class is not None:
self.threadpool_class = _import(self.threadpool_class)
self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
# pylint:disable=not-callable
self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
return self._threadpool
def _set_threadpool(self, value):
......
src/gevent/os.py
View file @ 0a0c16fb
......@@ -46,6 +46,7 @@ from __future__ import absolute_import
import os
import sys
from gevent.hub import get_hub, reinit
from gevent._config import config
from gevent._compat import PY3
from gevent._util import copy_globals
import errno
......@@ -418,7 +419,7 @@ if hasattr(os, 'fork'):
__extensions__.append('forkpty_and_watch')
# Watch children by default
if not os.getenv('GEVENT_NOWAITPID'):
if not config.disable_watch_children:
# Broken out into separate functions instead of simple name aliases
# for documentation purposes.
def fork(*args, **kwargs):
......
src/gevent/resolver/ares.py
View file @ 0a0c16fb
......@@ -27,6 +27,9 @@ from gevent.socket import SOCK_DGRAM
from gevent.socket import SOCK_RAW
from gevent.socket import AI_NUMERICHOST
from gevent._config import config
from gevent._config import AresSettingMixin
from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
from . import _lookup_port as lookup_port
from . import _resolve_special
......@@ -84,12 +87,11 @@ class Resolver(AbstractResolver):
hub = get_hub()
self.hub = hub
if use_environ:
for key in os.environ:
if key.startswith('GEVENTARES_'):
name = key[11:].lower()
if name:
value = os.environ[key]
kwargs.setdefault(name, value)
for setting in config.settings.values():
if isinstance(setting, AresSettingMixin):
value = setting.get()
if value is not None:
kwargs.setdefault(setting.kwarg_name, value)
self.ares = self.ares_class(hub.loop, **kwargs)
self.pid = os.getpid()
self.params = kwargs
......
src/gevent/resolver/dnspython.py
View file @ 0a0c16fb
......@@ -253,7 +253,9 @@ _is_ipv4_addr = _is_addr
def _is_ipv6_addr(host):
# Return True if host is a valid IPv6 address
host = host.split('%', 1)[0] if host else host
if host:
s = '%' if isinstance(host, str) else b'%'
host = host.split(s, 1)[0]
return _is_addr(host, dns.ipv6.inet_aton)
class HostsFile(object):
......
src/gevent/select.py
View file @ 0a0c16fb
......@@ -11,7 +11,6 @@ from gevent.hub import get_hub
from gevent.hub import sleep as _g_sleep
from gevent._compat import integer_types
from gevent._compat import iteritems
from gevent._compat import itervalues
from gevent._util import copy_globals
from gevent._util import _NONE
......@@ -200,7 +199,11 @@ if original_poll is not None:
.. versionadded:: 1.1b1
"""
def __init__(self):
self.fds = {} # {int -> watcher}
# {int -> flags}
# We can't keep watcher objects in here because people commonly
# just drop the poll object when they're done, without calling
# unregister(). dnspython does this.
self.fds = {}
self.loop = get_hub().loop
def register(self, fd, eventmask=_NONE):
......@@ -216,12 +219,7 @@ if original_poll is not None:
# that. Should we raise an error?
fileno = get_fileno(fd)
if fileno in self.fds:
self.fds[fileno].close()
watcher = self.loop.io(fileno, flags)
watcher.priority = self.loop.MAXPRI
self.fds[fileno] = watcher
self.fds[fileno] = flags
def modify(self, fd, eventmask):
self.register(fd, eventmask)
......@@ -234,16 +232,23 @@ if original_poll is not None:
File descriptors that are closed are reported with POLLNVAL.
"""
result = PollResult()
watchers = []
io = self.loop.io
MAXPRI = self.loop.MAXPRI
try:
for fd, watcher in iteritems(self.fds):
for fd, flags in iteritems(self.fds):
watcher = io(fd, flags)
watchers.append(watcher)
watcher.priority = MAXPRI
watcher.start(result.add_event, fd, pass_events=True)
if timeout is not None and timeout > -1:
timeout /= 1000.0
result.event.wait(timeout=timeout)
return list(result.events)
finally:
for awatcher in itervalues(self.fds):
for awatcher in watchers:
awatcher.stop()
awatcher.close()
def unregister(self, fd):
"""
......@@ -254,8 +259,6 @@ if original_poll is not None:
library. Previously gevent did nothing.
"""
fileno = get_fileno(fd)
io = self.fds[fileno]
io.close()
del self.fds[fileno]
del original_poll
src/greentest/greentest/__init__.py
View file @ 0a0c16fb
......@@ -58,6 +58,7 @@ from greentest.sysinfo import CONN_ABORTED_ERRORS
from greentest.skipping import skipOnWindows
from greentest.skipping import skipOnAppVeyor
from greentest.skipping import skipOnCI
from greentest.skipping import skipOnPyPy3OnCI
from greentest.skipping import skipOnPyPy
from greentest.skipping import skipOnPyPy3
......@@ -100,8 +101,6 @@ from greentest.openfiles import get_number_open_files
from greentest.openfiles import get_open_files
from greentest.testcase import TestCase
from greentest.timing import AbstractGenericGetTestCase as GenericGetTestCase
from greentest.timing import AbstractGenericWaitTestCase as GenericWaitTestCase
from greentest.modules import walk_modules
......
src/greentest/greentest/leakcheck.py
View file @ 0a0c16fb
......@@ -32,43 +32,44 @@ def ignores_leakcheck(func):
func.ignore_leakcheck = True
return func
# Some builtin things that we ignore
IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType)
try:
callback_kind = gevent.core.callback
except AttributeError:
# Must be using FFI.
from gevent._ffi.callback import callback as callback_kind
def _type_hist():
d = collections.defaultdict(int)
for x in gc.get_objects():
k = type(x)
if k in IGNORED_TYPES:
continue
if k == callback_kind and x.callback is None and x.args is None:
# these represent callbacks that have been stopped, but
# the event loop hasn't cycled around to run them. The only
# known cause of this is killing greenlets before they get a chance
# to run for the first time.
continue
d[k] += 1
return d
def _report_diff(a, b):
diff_lines = []
for k, v in sorted(a.items(), key=lambda i: i[0].__name__):
if b[k] != v:
diff_lines.append("%s: %s != %s" % (k, v, b[k]))
if not diff_lines:
return None
diff = '\n'.join(diff_lines)
return diff
def wrap_refcount(method):
if getattr(method, 'ignore_leakcheck', False):
return method
# Some builtin things that we ignore
IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType)
try:
callback_kind = gevent.core.callback
except AttributeError:
# Must be using FFI.
from gevent._ffi.callback import callback as callback_kind
def type_hist():
d = collections.defaultdict(int)
for x in gc.get_objects():
k = type(x)
if k in IGNORED_TYPES:
continue
if k == callback_kind and x.callback is None and x.args is None:
# these represent callbacks that have been stopped, but
# the event loop hasn't cycled around to run them. The only
# known cause of this is killing greenlets before they get a chance
# to run for the first time.
continue
d[k] += 1
return d
def report_diff(a, b):
diff_lines = []
for k, v in sorted(a.items(), key=lambda i: i[0].__name__):
if b[k] != v:
diff_lines.append("%s: %s != %s" % (k, v, b[k]))
if not diff_lines:
return None
diff = '\n'.join(diff_lines)
return diff
@wraps(method)
def wrapper(self, *args, **kwargs): # pylint:disable=too-many-branches
......@@ -78,25 +79,33 @@ def wrap_refcount(method):
deltas = []
d = None
gc.disable()
# The very first time we are called, we have already been
# self.setUp() by the test runner, so we don't need to do it again.
needs_setUp = False
try:
while True:
# Grab current snapshot
hist_before = type_hist()
hist_before = _type_hist()
d = sum(hist_before.values())
self.setUp()
if needs_setUp:
self.setUp()
self.skipTearDown = False
try:
method(self, *args, **kwargs)
finally:
self.tearDown()
self.skipTearDown = True
needs_setUp = True
# Grab post snapshot
if 'urlparse' in sys.modules:
sys.modules['urlparse'].clear_cache()
if 'urllib.parse' in sys.modules:
sys.modules['urllib.parse'].clear_cache()
hist_after = type_hist()
hist_after = _type_hist()
d = sum(hist_after.values()) - d
deltas.append(d)
......@@ -119,7 +128,7 @@ def wrap_refcount(method):
elif len(deltas) >= 4 and sum(deltas[-4:]) == 0:
break
elif len(deltas) >= 3 and deltas[-1] > 0 and deltas[-1] == deltas[-2] and deltas[-2] == deltas[-3]:
diff = report_diff(hist_before, hist_after)
diff = _report_diff(hist_before, hist_after)
raise AssertionError('refcount increased by %r\n%s' % (deltas, diff))
# OK, we don't know for sure yet. Let's search for more
if sum(deltas[-3:]) <= 0 or sum(deltas[-4:]) <= 0 or deltas[-4:].count(0) >= 2:
......@@ -128,9 +137,10 @@ def wrap_refcount(method):
else:
limit = 7
if len(deltas) >= limit:
raise AssertionError('refcount increased by %r\n%s' % (deltas, report_diff(hist_before, hist_after)))
raise AssertionError('refcount increased by %r\n%s'
% (deltas,
_report_diff(hist_before, hist_after)))
finally:
gc.enable()
self.skipTearDown = True
return wrapper
src/greentest/greentest/params.py
View file @ 0a0c16fb
......@@ -23,9 +23,11 @@ from greentest.sysinfo import PY3
from greentest.sysinfo import PYPY
from greentest.sysinfo import WIN
from greentest.sysinfo import LIBUV
from greentest.sysinfo import OSX
from greentest.sysinfo import RUNNING_ON_TRAVIS
from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
from greentest.sysinfo import RESOLVER_ARES
# Travis is slow and overloaded; Appveyor used to be faster, but
......@@ -60,6 +62,11 @@ if RUNNING_ON_TRAVIS:
# connected to with the same error.
DEFAULT_BIND_ADDR = '127.0.0.1'
if RESOLVER_ARES and OSX:
# Ares likes to raise "malformed domain name" on '', at least
# on OS X
DEFAULT_BIND_ADDR = '127.0.0.1'
# For in-process sockets
DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
......
src/greentest/greentest/skipping.py
View file @ 0a0c16fb
......@@ -23,13 +23,12 @@ import unittest
from greentest import sysinfo
def _identity(f):
return f
def _do_not_skip(reason):
assert reason
def dec(f):
return f
return dec
return _identity
if sysinfo.WIN:
......@@ -50,6 +49,11 @@ if sysinfo.RUNNING_ON_APPVEYOR:
else:
skipOnAppVeyor = _do_not_skip
if sysinfo.RUNNING_ON_CI:
skipOnCI = unittest.skip
else:
skipOnCI = _do_not_skip
if sysinfo.PYPY3 and sysinfo.RUNNING_ON_CI:
# Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
skipOnPyPy3OnCI = unittest.skip
......
src/greentest/greentest/sysinfo.py
View file @ 0a0c16fb
......@@ -30,8 +30,8 @@ LINUX = sys.platform.startswith('linux')
OSX = sys.platform == 'darwin'
# XXX: Formalize this better
LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' or (PYPY and WIN) or hasattr(gevent.core, 'libuv')
CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY
LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '')
if '--debug-greentest' in sys.argv:
sys.argv.remove('--debug-greentest')
......
src/greentest/greentest/testcase.py
View file @ 0a0c16fb
......@@ -95,7 +95,9 @@ class TestCaseMetaClass(type):
if sysinfo.RUN_LEAKCHECKS and timeout is not None:
timeout *= 6
check_totalrefcount = _get_class_attr(classDict, bases, 'check_totalrefcount', True)
error_fatal = _get_class_attr(classDict, bases, 'error_fatal', True)
uses_handle_error = _get_class_attr(classDict, bases, 'uses_handle_error', True)
# Python 3: must copy, we mutate the classDict. Interestingly enough,
# it doesn't actually error out, but under 3.6 we wind up wrapping
# and re-wrapping the same items over and over and over.
......@@ -108,7 +110,8 @@ class TestCaseMetaClass(type):
error_fatal = getattr(value, 'error_fatal', error_fatal)
if error_fatal:
value = errorhandler.wrap_error_fatal(value)
value = errorhandler.wrap_restore_handle_error(value)
if uses_handle_error:
value = errorhandler.wrap_restore_handle_error(value)
if check_totalrefcount and sysinfo.RUN_LEAKCHECKS:
value = leakcheck.wrap_refcount(value)
classDict[key] = value
......@@ -122,6 +125,7 @@ class TestCase(TestCaseMetaClass("NewBase", (TimeAssertMixin, BaseTestCase,), {}
switch_expected = 'default'
error_fatal = True
uses_handle_error = True
close_on_teardown = ()
def run(self, *args, **kwargs):
......
src/greentest/greentest/timing.py
View file @ 0a0c16fb
......@@ -58,6 +58,7 @@ class _DelayWaitMixin(object):
# otherwise it's the raw number
seconds = getattr(timeout, 'seconds', timeout)
gevent.get_hub().loop.update_now()
start = time.time()
try:
result = self.wait(timeout)
......
src/greentest/test___config.py 0 → 100644
View file @ 0a0c16fb
# Copyright 2018 gevent contributors. See LICENSE for details.
import os
import unittest
import sys
from gevent import _config
class TestResolver(unittest.TestCase):
old_resolver = None
def setUp(self):
if 'GEVENT_RESOLVER' in os.environ:
self.old_resolver = os.environ['GEVENT_RESOLVER']
del os.environ['GEVENT_RESOLVER']
def tearDown(self):
if self.old_resolver:
os.environ['GEVENT_RESOLVER'] = self.old_resolver
def test_key(self):
self.assertEqual(_config.Resolver.environment_key, 'GEVENT_RESOLVER')
def test_default(self):
from gevent.resolver.thread import Resolver
conf = _config.Resolver()
self.assertEqual(conf.get(), Resolver)
def test_env(self):
from gevent.resolver.blocking import Resolver
os.environ['GEVENT_RESOLVER'] = 'foo,bar,block,dnspython'
conf = _config.Resolver()
self.assertEqual(conf.get(), Resolver)
os.environ['GEVENT_RESOLVER'] = 'dnspython'
# The existing value is unchanged
self.assertEqual(conf.get(), Resolver)
# A new object reflects it
conf = _config.Resolver()
from gevent.resolver.dnspython import Resolver as DResolver
self.assertEqual(conf.get(), DResolver)
def test_set_str_long(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set('gevent.resolver.blocking.Resolver')
self.assertEqual(conf.get(), Resolver)
def test_set_str_short(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set('block')
self.assertEqual(conf.get(), Resolver)
def test_set_class(self):
from gevent.resolver.blocking import Resolver
conf = _config.Resolver()
conf.set(Resolver)
self.assertEqual(conf.get(), Resolver)
def test_set_through_config(self):
from gevent.resolver.thread import Resolver as Default
from gevent.resolver.blocking import Resolver
conf = _config.Config()
self.assertEqual(conf.resolver, Default)
conf.resolver = 'block'
self.assertEqual(conf.resolver, Resolver)
class TestFunctions(unittest.TestCase):
def test_validate_bool(self):
self.assertTrue(_config.validate_bool('on'))
self.assertTrue(_config.validate_bool('1'))
self.assertFalse(_config.validate_bool('off'))
self.assertFalse(_config.validate_bool('0'))
self.assertFalse(_config.validate_bool(''))
with self.assertRaises(ValueError):
_config.validate_bool(' hmm ')
def test_validate_invalid(self):
with self.assertRaises(ValueError):
_config.validate_invalid(self)
class TestConfig(unittest.TestCase):
def test__dir__(self):
self.assertEqual(sorted(_config.config.settings),
sorted(dir(_config.config)))
def test_getattr(self):
# Bypass the property that might be set here
self.assertIsNotNone(_config.config.__getattr__('resolver'))
def test__getattr__invalid(self):
with self.assertRaises(AttributeError):
getattr(_config.config, 'no_such_setting')
def test_set_invalid(self):
with self.assertRaises(AttributeError):
_config.config.set('no such setting', True)
class TestImportableSetting(unittest.TestCase):
assertRaisesRegex = getattr(unittest.TestCase, 'assertRaisesRegex',
unittest.TestCase.assertRaisesRegexp)
def test_empty_list(self):
i = _config.ImportableSetting()
with self.assertRaisesRegex(ImportError,
"Cannot import from empty list"):
i._import([])
def test_path(self):
import warnings
i = _config.ImportableSetting()
path = list(sys.path)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
with self.assertRaisesRegex(ImportError,
"Cannot import 'no_such_module'"):
i._import('foo/bar/gevent.no_such_module')
# We restored the path
self.assertEqual(path, sys.path)
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, DeprecationWarning)
self.assertIn('Absolute paths', str(w[0].message))
def test_non_string(self):
i = _config.ImportableSetting()
self.assertIs(i._import(self), self)
if __name__ == '__main__':
unittest.main()
src/greentest/test___example_servers.py
View file @ 0a0c16fb
......@@ -9,8 +9,8 @@ from unittest import SkipTest
import socket
import ssl
import greentest
from greentest import DEFAULT_XPC_SOCKET_TIMEOUT
from greentest import main
from greentest import util
from greentest import params
......@@ -101,6 +101,7 @@ class Test_wsgiserver_ssl(Test_wsgiserver):
ssl_ctx = ssl._create_unverified_context()
@greentest.skipOnCI("Timing issues sometimes lead to a connection refused")
class Test_webproxy(Test_wsgiserver):
server = 'webproxy.py'
......@@ -136,4 +137,4 @@ class Test_webproxy(Test_wsgiserver):
if __name__ == '__main__':
main()
greentest.main()
src/greentest/test__api.py
View file @ 0a0c16fb
......@@ -105,9 +105,10 @@ class TestTimers(greentest.TestCase):
gevent.sleep(0.02)
gevent.spawn(func)
assert lst == [1], lst
gevent.sleep(0.03)
assert lst == [], lst
self.assertEqual(lst, [1])
gevent.sleep(0.1)
self.assertEqual(lst, [])
def test_spawn_is_not_cancelled(self):
lst = [1]
......@@ -116,8 +117,8 @@ class TestTimers(greentest.TestCase):
gevent.spawn(lst.pop)
# exiting immediately, but self.lst.pop must be called
gevent.spawn(func)
gevent.sleep(0.01)
assert lst == [], lst
gevent.sleep(0.1)
self.assertEqual(lst, [])
if __name__ == '__main__':
......
src/greentest/test__core_timer.py
View file @ 0a0c16fb
from __future__ import print_function
from gevent import core
from gevent import config
from greentest import TestCase
from greentest import main
......@@ -9,15 +9,23 @@ from greentest.sysinfo import CFFI_BACKEND
class Test(TestCase):
__timeout__ = LARGE_TIMEOUT
repeat = 0
def setUp(self):
self.called = []
self.loop = core.loop(default=True)
self.loop = config.loop(default=False)
self.timer = self.loop.timer(0.001, repeat=self.repeat)
def tearDown(self):
def cleanup(self):
# cleanup instead of tearDown to cooperate well with
# leakcheck.py
self.timer.close()
# cycle the loop so libuv close callbacks fire
self.loop.run()
self.loop.destroy()
self.loop = None
self.timer = None
def f(self, x=None):
self.called.append(1)
......@@ -67,6 +75,8 @@ class TestAgain(Test):
self.assertEqual(x.args, (x,))
# XXX: On libev, this takes 1 second. On libuv,
# it takes the expected time.
self.loop.run()
self.assertEqual(self.called, [1])
......
src/greentest/test__event.py
View file @ 0a0c16fb
......@@ -6,11 +6,13 @@ from gevent.event import Event, AsyncResult
import greentest
from greentest.skipping import skipUnderCoverage
from greentest.six import xrange
from greentest.timing import AbstractGenericGetTestCase
from greentest.timing import AbstractGenericWaitTestCase
DELAY = 0.01
class TestEventWait(greentest.GenericWaitTestCase):
class TestEventWait(AbstractGenericWaitTestCase):
def wait(self, timeout):
Event().wait(timeout=timeout)
......@@ -19,7 +21,7 @@ class TestEventWait(greentest.GenericWaitTestCase):
str(Event())
class TestWaitEvent(greentest.GenericWaitTestCase):
class TestWaitEvent(AbstractGenericWaitTestCase):
def wait(self, timeout):
gevent.wait([Event()], timeout=timeout)
......@@ -58,19 +60,19 @@ class TestWaitEvent(greentest.GenericWaitTestCase):
gevent.spawn(waiter).join()
class TestAsyncResultWait(greentest.GenericWaitTestCase):
class TestAsyncResultWait(AbstractGenericWaitTestCase):
def wait(self, timeout):
AsyncResult().wait(timeout=timeout)
class TestWaitAsyncResult(greentest.GenericWaitTestCase):
class TestWaitAsyncResult(AbstractGenericWaitTestCase):
def wait(self, timeout):
gevent.wait([AsyncResult()], timeout=timeout)
class TestAsyncResultGet(greentest.GenericGetTestCase):
class TestAsyncResultGet(AbstractGenericGetTestCase):
def wait(self, timeout):
AsyncResult().get(timeout=timeout)
......@@ -232,5 +234,8 @@ class TestWait_count2(TestWait):
count = 2
del AbstractGenericGetTestCase
del AbstractGenericWaitTestCase
if __name__ == '__main__':
greentest.main()
src/greentest/test__greenlet.py
View file @ 0a0c16fb
......@@ -19,7 +19,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
import sys
import greentest
import gevent
import re
......@@ -28,6 +28,9 @@ from gevent import greenlet
from gevent.event import AsyncResult
from gevent.queue import Queue, Channel
from greentest.timing import AbstractGenericWaitTestCase
from greentest.timing import AbstractGenericGetTestCase
DELAY = 0.01
greentest.TestCase.error_fatal = False
......@@ -38,25 +41,13 @@ class ExpectedError(greentest.ExpectedException):
class TestLink(greentest.TestCase):
def assertRaises(self, err, func, *args, **kwargs):
try:
result = func(*args, **kwargs)
except:
ex = sys.exc_info()[1]
if ex is err:
return
if isinstance(ex, err):
return
raise
raise AssertionError('%s not raised, returned %r' % (err, result))
def test_link_to_asyncresult(self):
p = gevent.spawn(lambda: 100)
event = AsyncResult()
p.link(event)
self.assertEqual(event.get(), 100)
for i in range(3):
for _ in range(3):
event2 = AsyncResult()
p.link(event2)
self.assertEqual(event2.get(), 100)
......@@ -66,12 +57,17 @@ class TestLink(greentest.TestCase):
p = gevent.spawn(lambda: getcurrent().throw(err))
event = AsyncResult()
p.link(event)
self.assertRaises(err, event.get)
with self.assertRaises(ExpectedError) as exc:
event.get()
self.assertIs(exc.exception, err)
for i in range(3):
for _ in range(3):
event2 = AsyncResult()
p.link(event2)
self.assertRaises(err, event2.get)
with self.assertRaises(ExpectedError) as exc:
event2.get()
self.assertIs(exc.exception, err)
def test_link_to_queue(self):
p = gevent.spawn(lambda: 100)
......@@ -79,7 +75,7 @@ class TestLink(greentest.TestCase):
p.link(q.put)
self.assertEqual(q.get().get(), 100)
for i in range(3):
for _ in range(3):
p.link(q.put)
self.assertEqual(q.get().get(), 100)
......@@ -125,6 +121,8 @@ class TestUnlink(greentest.TestCase):
class LinksTestCase(greentest.TestCase):
link_method = None
def link(self, p, listener=None):
getattr(p, self.link_method)(listener)
......@@ -169,30 +167,33 @@ def sleep0():
class TestReturn_link(LinksTestCase):
link_method = 'link'
p = None
def cleanup(self):
while self.p._links:
self.p._links.pop()
self.p = None
def test_return(self):
self.p = gevent.spawn(return25)
for _ in range(3):
self._test_return(self.p, 25, sleep0)
self._test_return(self.p, 25)
self.p.kill()
def _test_return(self, p, result, action):
def _test_return(self, p, result):
event, queue, callback_flag = self.set_links(p)
# stuff that will time out because there's no unhandled exception:
xxxxx = self.set_links_timeout(p.link_exception)
sleep(DELAY * 2)
assert not p, p
self.assertFalse(p)
self.assertEqual(event.get(), result)
self.assertEqual(queue.get().get(), result)
sleep(DELAY)
assert not callback_flag, callback_flag
self.assertFalse(callback_flag)
self.check_timed_out(*xxxxx)
......@@ -202,13 +203,14 @@ class TestReturn_link(LinksTestCase):
p.kill()
sleep(DELAY)
assert not p, p
self.assertFalse(p)
assert isinstance(event.get(), greenlet.GreenletExit), event.get()
assert isinstance(queue.get().get(), greenlet.GreenletExit), queue.get().get()
self.assertIsInstance(event.get(), greenlet.GreenletExit)
self.assertIsInstance(queue.get().get(), greenlet.GreenletExit)
sleep(DELAY)
assert not callback_flag, callback_flag
self.assertFalse(callback_flag)
self.check_timed_out(*xxxxx)
......@@ -240,7 +242,7 @@ class TestRaise_link(LinksTestCase):
self.check_timed_out(*xxxxx)
def test_raise(self):
p = self.p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_raise')))
p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_raise')))
for _ in range(3):
self._test_raise(p)
......@@ -320,22 +322,22 @@ class TestStuff(greentest.TestCase):
p = gevent.spawn(lambda: 5)
results = []
def listener1(*args):
def listener1(*_args):
results.append(10)
raise ExpectedError('listener1')
def listener2(*args):
def listener2(*_args):
results.append(20)
raise ExpectedError('listener2')
def listener3(*args):
def listener3(*_args):
raise ExpectedError('listener3')
p.link(listener1)
p.link(listener2)
p.link(listener3)
sleep(DELAY * 10)
assert results in [[10, 20], [20, 10]], results
self.assertIn(results, [[10, 20], [20, 10]])
p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_multiple_listeners_error')))
results = []
......@@ -343,7 +345,7 @@ class TestStuff(greentest.TestCase):
p.link(listener2)
p.link(listener3)
sleep(DELAY * 10)
assert results in [[10, 20], [20, 10]], results
self.assertIn(results, [[10, 20], [20, 10]])
class Results(object):
......@@ -360,10 +362,10 @@ class TestStuff(greentest.TestCase):
self.results.append(5)
raise ExpectedError('listener2')
def listener3(self, p):
def listener3(self, _p):
raise ExpectedError('listener3')
def _test_multiple_listeners_error_unlink(self, p, link):
def _test_multiple_listeners_error_unlink(self, _p, link):
# notification must not happen after unlink even
# though notification process has been already started
results = self.Results()
......@@ -388,20 +390,8 @@ class TestStuff(greentest.TestCase):
gevent.spawn(e.set, 6)
self._test_multiple_listeners_error_unlink(e, e.rawlink)
def test_killing_unlinked(self):
e = AsyncResult()
def func():
try:
raise ExpectedError('test_killing_unlinked')
except:
e.set_exception(sys.exc_info()[1])
gevent.sleep(0)
sleep(DELAY)
def dummy_test_func(*args):
def dummy_test_func(*_args):
pass
......@@ -436,7 +426,7 @@ class TestStr(greentest.TestCase):
self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
class TestJoin(greentest.GenericWaitTestCase):
class TestJoin(AbstractGenericWaitTestCase):
def wait(self, timeout):
g = gevent.spawn(gevent.sleep, 10)
......@@ -446,7 +436,7 @@ class TestJoin(greentest.GenericWaitTestCase):
g.kill()
class TestGet(greentest.GenericGetTestCase):
class TestGet(AbstractGenericGetTestCase):
def wait(self, timeout):
g = gevent.spawn(gevent.sleep, 10)
......@@ -456,7 +446,7 @@ class TestGet(greentest.GenericGetTestCase):
g.kill()
class TestJoinAll0(greentest.GenericWaitTestCase):
class TestJoinAll0(AbstractGenericWaitTestCase):
g = gevent.Greenlet()
......@@ -464,7 +454,7 @@ class TestJoinAll0(greentest.GenericWaitTestCase):
gevent.joinall([self.g], timeout=timeout)
class TestJoinAll(greentest.GenericWaitTestCase):
class TestJoinAll(AbstractGenericWaitTestCase):
def wait(self, timeout):
g = gevent.spawn(gevent.sleep, 10)
......@@ -550,7 +540,7 @@ class TestBasic(greentest.TestCase):
def func(delay, return_value=4):
gevent.sleep(delay)
error = ExpectedError('test_error_exit')
error.myattr = return_value
setattr(error, 'myattr', return_value)
raise error
g = gevent.Greenlet(func, 0.001, return_value=5)
......@@ -558,13 +548,14 @@ class TestBasic(greentest.TestCase):
g.rawlink(link_test.append)
g.start()
gevent.sleep(0.1)
assert not g
assert g.dead
assert not g.started
assert g.ready()
assert not g.successful()
assert g.value is None # not changed
assert g.exception.myattr == 5
self.assertFalse(g)
self.assertTrue(g.dead)
self.assertFalse(g.started)
self.assertTrue(g.ready())
self.assertFalse(g.successful())
self.assertIsNone(g.value) # not changed
self.assertEqual(g.exception.myattr, 5)
assert link_test == [g] or greentest.RUNNING_ON_APPVEYOR, link_test
def _assertKilled(self, g):
......@@ -594,7 +585,7 @@ class TestBasic(greentest.TestCase):
link_test = []
result = []
g = gevent.Greenlet(lambda: result.append(1))
g.link(lambda x: link_test.append(x))
g.link(link_test.append)
self._test_kill(g, block=block)
assert not result
assert link_test == [g]
......@@ -609,7 +600,7 @@ class TestBasic(greentest.TestCase):
result = []
link_test = []
g = gevent.Greenlet(lambda: result.append(1))
g.link(lambda x: link_test.append(x))
g.link(link_test.append)
g.start()
self._test_kill(g, block=block)
assert not result, result
......@@ -625,7 +616,7 @@ class TestBasic(greentest.TestCase):
result = []
link_test = []
g = gevent.Greenlet(lambda: result.append(1))
g.link(lambda x: link_test.append(x))
g.link(link_test.append)
g.start_later(1)
self._test_kill(g, block=block)
assert not result
......@@ -639,7 +630,7 @@ class TestBasic(greentest.TestCase):
def _test_kill_running(self, block):
link_test = []
g = gevent.spawn(gevent.sleep, 10)
g.link(lambda x: link_test.append(x))
g.link(link_test.append)
self._test_kill(g, block=block)
gevent.sleep(0.01)
assert link_test == [g]
......@@ -707,5 +698,9 @@ class TestRef(greentest.TestCase):
X = object()
del AbstractGenericGetTestCase
del AbstractGenericWaitTestCase
if __name__ == '__main__':
greentest.main()
src/greentest/test__hub.py
View file @ 0a0c16fb
......@@ -20,6 +20,7 @@
# THE SOFTWARE.
import greentest
import greentest.timing
import time
import re
import gevent
......@@ -71,7 +72,7 @@ class TestExceptionInMainloop(greentest.TestCase):
class TestSleep(greentest.GenericWaitTestCase):
class TestSleep(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
gevent.sleep(timeout)
......@@ -80,7 +81,7 @@ class TestSleep(greentest.GenericWaitTestCase):
gevent.sleep(0)
class TestWaiterGet(greentest.GenericWaitTestCase):
class TestWaiterGet(greentest.timing.AbstractGenericWaitTestCase):
def setUp(self):
super(TestWaiterGet, self).setUp()
......
src/greentest/test__pool.py
View file @ 0a0c16fb
from time import time
import gevent
from gevent import pool
import gevent.pool
from gevent.event import Event
from gevent.queue import Queue
from gevent.timeout import Timeout
import greentest
import greentest.timing
import random
from greentest import ExpectedException
from greentest import six
......@@ -13,12 +14,12 @@ import unittest
class TestCoroutinePool(unittest.TestCase):
klass = pool.Pool
klass = gevent.pool.Pool
def test_apply_async(self):
done = Event()
def some_work(x):
def some_work(_):
done.set()
pool = self.klass(2)
......@@ -125,18 +126,18 @@ class TestCoroutinePool(unittest.TestCase):
pool.join()
def crash(*args, **kw):
def crash(*_args, **_kw):
raise RuntimeError("Whoa")
class FakeFile(object):
def write(*args):
def write(self, *_args):
raise RuntimeError('Whaaa')
class PoolBasicTests(greentest.TestCase):
klass = pool.Pool
klass = gevent.pool.Pool
def test_execute_async(self):
p = self.klass(size=2)
......@@ -208,7 +209,7 @@ class PoolBasicTests(greentest.TestCase):
second = gevent.spawn(gevent.sleep, 1000)
try:
p.add(first)
with self.assertRaises(pool.PoolFull):
with self.assertRaises(gevent.pool.PoolFull):
p.add(second, blocking=False)
finally:
second.kill()
......@@ -223,7 +224,7 @@ class PoolBasicTests(greentest.TestCase):
second = gevent.spawn(gevent.sleep, 1000)
try:
p.add(first)
with self.assertRaises(pool.PoolFull):
with self.assertRaises(gevent.pool.PoolFull):
p.add(second, timeout=0.100)
finally:
second.kill()
......@@ -238,7 +239,7 @@ class PoolBasicTests(greentest.TestCase):
second = gevent.Greenlet(gevent.sleep, 1000)
try:
p.add(first)
with self.assertRaises(pool.PoolFull):
with self.assertRaises(gevent.pool.PoolFull):
p.start(second, timeout=0.100)
finally:
second.kill()
......@@ -304,13 +305,13 @@ if greentest.PYPY and greentest.WIN:
elif greentest.RUNNING_ON_CI or greentest.EXPECT_POOR_TIMER_RESOLUTION:
LARGE_RANGE = 100
class TestPool(greentest.TestCase):
class TestPool(greentest.TestCase): # pylint:disable=too-many-public-methods
__timeout__ = greentest.LARGE_TIMEOUT
size = 1
def setUp(self):
greentest.TestCase.setUp(self)
self.pool = pool.Pool(self.size)
self.pool = gevent.pool.Pool(self.size)
def cleanup(self):
self.pool.join()
......@@ -438,17 +439,11 @@ class TestPool(greentest.TestCase):
running = [0]
def short_running_func(i, j):
def short_running_func(i, _j):
running[0] += 1
return i
# Send two iterables to make sure varargs and kwargs are handled
# correctly
for meth in self.pool.imap_unordered, self.pool.imap:
running[0] = 0
mapping = meth(short_running_func, iterable, iterable,
maxsize=1)
def make_reader(mapping):
# Simulate a long running reader. No matter how many workers
# we have, we will never have a queue more than size 1
def reader():
......@@ -459,7 +454,16 @@ class TestPool(greentest.TestCase):
gevent.sleep(0.01)
self.assertTrue(len(mapping.queue) <= 2, len(mapping.queue))
return result
return reader
# Send two iterables to make sure varargs and kwargs are handled
# correctly
for meth in self.pool.imap_unordered, self.pool.imap:
running[0] = 0
mapping = meth(short_running_func, iterable, iterable,
maxsize=1)
reader = make_reader(mapping)
l = reader()
self.assertEqual(sorted(l), iterable)
......@@ -484,16 +488,16 @@ class TestPool0(greentest.TestCase):
size = 0
def test_wait_full(self):
p = pool.Pool(size=0)
p = gevent.pool.Pool(size=0)
self.assertEqual(0, p.free_count())
self.assertTrue(p.full())
self.assertEqual(0, p.wait_available(timeout=0.01))
class TestJoinSleep(greentest.GenericWaitTestCase):
class TestJoinSleep(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
p = pool.Pool()
p = gevent.pool.Pool()
g = p.spawn(gevent.sleep, 10)
try:
p.join(timeout=timeout)
......@@ -501,10 +505,10 @@ class TestJoinSleep(greentest.GenericWaitTestCase):
g.kill()
class TestJoinSleep_raise_error(greentest.GenericWaitTestCase):
class TestJoinSleep_raise_error(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
p = pool.Pool()
p = gevent.pool.Pool()
g = p.spawn(gevent.sleep, 10)
try:
p.join(timeout=timeout, raise_error=True)
......@@ -516,7 +520,7 @@ class TestJoinEmpty(greentest.TestCase):
switch_expected = False
def test(self):
p = pool.Pool()
p = gevent.pool.Pool()
res = p.join()
self.assertTrue(res, "empty should return true")
......@@ -525,7 +529,7 @@ class TestSpawn(greentest.TestCase):
switch_expected = True
def test(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
self.assertEqual(len(p), 0)
p.spawn(gevent.sleep, 0.1)
self.assertEqual(len(p), 1)
......@@ -535,7 +539,7 @@ class TestSpawn(greentest.TestCase):
self.assertEqual(len(p), 0)
def testSpawnAndWait(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
self.assertEqual(len(p), 0)
p.spawn(gevent.sleep, 0.1)
self.assertEqual(len(p), 1)
......@@ -555,12 +559,12 @@ class TestErrorInIterator(greentest.TestCase):
error_fatal = False
def test(self):
p = pool.Pool(3)
p = gevent.pool.Pool(3)
self.assertRaises(ExpectedException, p.map, lambda x: None, error_iter())
gevent.sleep(0.001)
def test_unordered(self):
p = pool.Pool(3)
p = gevent.pool.Pool(3)
def unordered():
return list(p.imap_unordered(lambda x: None, error_iter()))
......@@ -577,11 +581,11 @@ class TestErrorInHandler(greentest.TestCase):
error_fatal = False
def test_map(self):
p = pool.Pool(3)
p = gevent.pool.Pool(3)
self.assertRaises(ZeroDivisionError, p.map, divide_by, [1, 0, 2])
def test_imap(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
it = p.imap(divide_by, [1, 0, 2])
self.assertEqual(next(it), 1.0)
self.assertRaises(ZeroDivisionError, next, it)
......@@ -589,7 +593,7 @@ class TestErrorInHandler(greentest.TestCase):
self.assertRaises(StopIteration, next, it)
def test_imap_unordered(self):
p = pool.Pool(1)
p = gevent.pool.Pool(1)
it = p.imap_unordered(divide_by, [1, 0, 2])
self.assertEqual(next(it), 1.0)
self.assertRaises(ZeroDivisionError, next, it)
......
src/greentest/test__queue.py
View file @ 0a0c16fb
import greentest
from greentest import TestCase, main, GenericGetTestCase
from greentest import TestCase, main
import gevent
from gevent.hub import get_hub, LoopExit
from gevent import util
from gevent import queue
from gevent.queue import Empty, Full
from gevent.event import AsyncResult
from greentest.timing import AbstractGenericGetTestCase
# pylint:disable=too-many-ancestors
class TestQueue(TestCase):
......@@ -372,7 +374,7 @@ class TestJoinEmpty(TestCase):
q.join()
class TestGetInterrupt(GenericGetTestCase):
class TestGetInterrupt(AbstractGenericGetTestCase):
Timeout = Empty
......@@ -397,7 +399,7 @@ class TestGetInterruptChannel(TestGetInterrupt):
kind = queue.Channel
class TestPutInterrupt(GenericGetTestCase):
class TestPutInterrupt(AbstractGenericGetTestCase):
kind = queue.Queue
Timeout = Full
......@@ -430,7 +432,7 @@ class TestPutInterruptChannel(TestPutInterrupt):
return self.kind()
del GenericGetTestCase
del AbstractGenericGetTestCase
if __name__ == '__main__':
......
src/greentest/test__select.py
View file @ 0a0c16fb
......@@ -5,86 +5,87 @@ import errno
from gevent import select, socket
import gevent.core
import greentest
import greentest.timing
import unittest
class TestSelect(greentest.GenericWaitTestCase):
class TestSelect(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
select.select([], [], [], timeout)
if sys.platform != 'win32':
class TestSelectRead(greentest.GenericWaitTestCase):
@greentest.skipOnWindows("Cant select on files")
class TestSelectRead(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
r, w = os.pipe()
def wait(self, timeout):
r, w = os.pipe()
try:
select.select([r], [], [], timeout)
finally:
os.close(r)
os.close(w)
# Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606
@unittest.skipIf(sys.platform.startswith('freebsd'),
'skip because of a FreeBSD bug: kern/155606')
def test_errno(self):
# Backported from test_select.py in 3.4
with open(__file__, 'rb') as fp:
fd = fp.fileno()
fp.close()
try:
select.select([r], [], [], timeout)
finally:
os.close(r)
os.close(w)
# Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606
@unittest.skipIf(sys.platform.startswith('freebsd'),
'skip because of a FreeBSD bug: kern/155606')
def test_errno(self):
# Backported from test_select.py in 3.4
with open(__file__, 'rb') as fp:
fd = fp.fileno()
fp.close()
try:
select.select([fd], [], [], 0)
except OSError as err:
# Python 3
self.assertEqual(err.errno, errno.EBADF)
except select.error as err: # pylint:disable=duplicate-except
# Python 2 (select.error is OSError on py3)
self.assertEqual(err.args[0], errno.EBADF)
else:
self.fail("exception not raised")
if hasattr(select, 'poll'):
class TestPollRead(greentest.GenericWaitTestCase):
def wait(self, timeout):
# On darwin, the read pipe is reported as writable
# immediately, for some reason. So we carefully register
# it only for read events (the default is read and write)
r, w = os.pipe()
try:
poll = select.poll()
poll.register(r, select.POLLIN)
poll.poll(timeout * 1000)
finally:
poll.unregister(r)
os.close(r)
os.close(w)
def test_unregister_never_registered(self):
# "Attempting to remove a file descriptor that was
# never registered causes a KeyError exception to be
# raised."
poll = select.poll()
self.assertRaises(KeyError, poll.unregister, 5)
@unittest.skipIf(hasattr(gevent.core, 'libuv'),
"Depending on whether the fileno is reused or not this either crashes or does nothing."
"libuv won't open a watcher for a closed file on linux.")
def test_poll_invalid(self):
with open(__file__, 'rb') as fp:
fd = fp.fileno()
poll = select.poll()
poll.register(fd, select.POLLIN)
# Close after registering; libuv refuses to even
# create a watcher if it would get EBADF (so this turns into
# a test of whether or not we successfully initted the watcher).
fp.close()
result = poll.poll(0)
self.assertEqual(result, [(fd, select.POLLNVAL)]) # pylint:disable=no-member
select.select([fd], [], [], 0)
except OSError as err:
# Python 3
self.assertEqual(err.errno, errno.EBADF)
except select.error as err: # pylint:disable=duplicate-except
# Python 2 (select.error is OSError on py3)
self.assertEqual(err.args[0], errno.EBADF)
else:
self.fail("exception not raised")
@unittest.skipUnless(hasattr(select, 'poll'), "Needs poll")
@greentest.skipOnWindows("Cant poll on files")
class TestPollRead(greentest.timing.AbstractGenericWaitTestCase):
def wait(self, timeout):
# On darwin, the read pipe is reported as writable
# immediately, for some reason. So we carefully register
# it only for read events (the default is read and write)
r, w = os.pipe()
try:
poll = select.poll()
poll.register(r, select.POLLIN)
poll.poll(timeout * 1000)
finally:
poll.unregister(r)
os.close(r)
os.close(w)
def test_unregister_never_registered(self):
# "Attempting to remove a file descriptor that was
# never registered causes a KeyError exception to be
# raised."
poll = select.poll()
self.assertRaises(KeyError, poll.unregister, 5)
@unittest.skipIf(hasattr(gevent.core, 'libuv'),
"Depending on whether the fileno is reused or not this either crashes or does nothing."
"libuv won't open a watcher for a closed file on linux.")
def test_poll_invalid(self):
with open(__file__, 'rb') as fp:
fd = fp.fileno()
poll = select.poll()
poll.register(fd, select.POLLIN)
# Close after registering; libuv refuses to even
# create a watcher if it would get EBADF (so this turns into
# a test of whether or not we successfully initted the watcher).
fp.close()
result = poll.poll(0)
self.assertEqual(result, [(fd, select.POLLNVAL)]) # pylint:disable=no-member
class TestSelectTypes(greentest.TestCase):
......
src/greentest/test__socket.py
View file @ 0a0c16fb
......@@ -9,6 +9,7 @@ import unittest
import greentest
from functools import wraps
from greentest import six
from greentest import LARGE_TIMEOUT
# we use threading on purpose so that we can test both regular and gevent sockets with the same code
from threading import Thread as _Thread
......@@ -42,7 +43,7 @@ class TestTCP(greentest.TestCase):
__timeout__ = None
TIMEOUT_ERROR = socket.timeout
long_data = ", ".join([str(x) for x in range(20000)])
if six.PY3:
if not isinstance(long_data, bytes):
long_data = long_data.encode('ascii')
def setUp(self):
......@@ -341,7 +342,7 @@ def get_port():
class TestCreateConnection(greentest.TestCase):
__timeout__ = 5
__timeout__ = LARGE_TIMEOUT
def test_refuses(self):
with self.assertRaises(socket.error) as cm:
......
src/greentest/test__socket_dns.py
View file @ 0a0c16fb
......@@ -25,7 +25,9 @@ if getattr(resolver, 'pool', None) is not None:
from greentest.sysinfo import RESOLVER_NOT_SYSTEM
from greentest.sysinfo import RESOLVER_DNSPYTHON
from greentest.sysinfo import RESOLVER_ARES
from greentest.sysinfo import PY2
import greentest.timing
assert gevent_socket.gaierror is socket.gaierror
......@@ -444,6 +446,8 @@ class SanitizedHostsFile(HostsFile):
continue
yield name, addr
@greentest.skipIf(greentest.RUNNING_ON_TRAVIS and RESOLVER_ARES,
"This sometimes randomly fails on Travis with ares, beginning Feb 13, 2018")
class TestEtcHosts(TestCase):
MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
......@@ -567,7 +571,7 @@ add(TestInternational, u'президент.рф', 'russian',
add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
class TestInterrupted_gethostbyname(greentest.timing.AbstractGenericWaitTestCase):
# There are refs to a Waiter in the C code that don't go
# away yet; one gc may or may not do it.
......
src/greentest/tests_that_dont_use_resolver.txt
View file @ 0a0c16fb
......@@ -123,3 +123,5 @@ test_timeout.py
# implicitly for localhost, which is covered well enough
# elsewhere that we don't need to spend the 20s (*2)
test_asyncore.py
test___config.py
tox.ini
View file @ 0a0c16fb
......@@ -36,7 +36,7 @@ commands =
basepython =
python2.7
setenv =
GEVENT_CORE_CFFI_ONLY=1
GEVENT_LOOP=libev-cffi
commands =
make basictest
......@@ -44,7 +44,7 @@ commands =
basepython =
python2.7
setenv =
GEVENT_CORE_CFFI_ONLY=libuv
GEVENT_LOOP=libuv-cffi
commands =
make basictest
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7