Merge pull request #1096 from gevent/config-issue1090

Centralize configuration in a ``gevent.config`` object.

CHANGES.rst

@@ -46,6 +46,18 @@
   :mod:`time`, much like :mod:`gevent.socket` can be imported instead
   of :mod:`socket`.
 
+- Centralize all gevent configuration in an object at
+  ``gevent.config``, allowing for gevent to be configured through code
+  and not *necessarily* environment variables, and also provide a
+  centralized place for documentation. See :issue:`1090`.
+
+  - The new ``GEVENT_CORE_CFFI_ONLY`` environment variable has been
+    replaced with the pre-existing ``GEVENT_LOOP`` environment
+    variable. That variable may take the values ``libev-cext``,
+    ``libev-cffi``, or ``libuv-cffi``, (or be a list in preference
+    order, or be a dotted name; it may also be assigned to an
+    object in Python code at ``gevent.config.loop``).
+
 1.3a1 (2018-01-27)
 ==================
 

Makefile

@@ -45,46 +45,43 @@ prospector:
 lint: prospector
 
 test_prelim:
-	which ${PYTHON}
-	${PYTHON} --version
-	${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
-	${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
-	${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
-	make bench
+	@which ${PYTHON}
+	@${PYTHON} --version
+	@${PYTHON} -c 'import greenlet; print(greenlet, greenlet.__version__)'
+	@${PYTHON} -c 'import gevent.core; print(gevent.core.loop)'
+	@${PYTHON} -c 'import gevent.ares; print(gevent.ares)'
+	@make bench
 
 # Folding from https://github.com/travis-ci/travis-rubies/blob/9f7962a881c55d32da7c76baefc58b89e3941d91/build.sh#L38-L44
-# 	echo -e "travis_fold:start:${GEVENT_CORE_CFFI_ONLY}\033[33;1m${GEVENT_CORE_CFFI_ONLY}\033[0m"
-# Make calls /bin/echo, which doesn't support the -e option, which is part of the bash builtin.
-# we need a python script to do this, or possible the GNU make shell function
 
 basictest: test_prelim
-	${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
+	@${PYTHON} scripts/travis.py fold_start basictest "Running basic tests"
 	cd src/greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config known_failures.py --quiet
-	${PYTHON} scripts/travis.py fold_end basictest
+	@${PYTHON} scripts/travis.py fold_end basictest
 
 alltest: basictest
-	${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
+	@${PYTHON} scripts/travis.py fold_start ares "Running c-ares tests"
 	cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
-	${PYTHON} scripts/travis.py fold_end ares
-	${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
+	@${PYTHON} scripts/travis.py fold_end ares
+	@${PYTHON} scripts/travis.py fold_start dnspython "Running dnspython tests"
 	cd src/greentest && GEVENT_RESOLVER=dnspython ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt --quiet
-	${PYTHON} scripts/travis.py fold_end dnspython
+	@${PYTHON} scripts/travis.py fold_end dnspython
 # In the past, we included all test files that had a reference to 'subprocess'' somewhere in their
 # text. The monkey-patched stdlib tests were specifically included here.
 # However, we now always also test on AppVeyor (Windows) which only has GEVENT_FILE=thread,
 # so we can save a lot of CI time by reducing the set and excluding the stdlib tests without
 # losing any coverage. See the `threadfiletest` for what command used to run.
-	${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
+	@${PYTHON} scripts/travis.py fold_start thread "Running GEVENT_FILE=thread tests"
 	cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py test__*subprocess*.py --quiet
-	${PYTHON} scripts/travis.py fold_end thread
+	@${PYTHON} scripts/travis.py fold_end thread
 
 threadfiletest:
 	cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py `grep -l subprocess test_*.py` --quiet
 
 allbackendtest:
-	${PYTHON} scripts/travis.py fold_start default "Testing default backend"
-	GEVENT_CORE_CFFI_ONLY= GEVENTTEST_COVERAGE=1 make alltest
-	${PYTHON} scripts/travis.py fold_end default
+	@${PYTHON} scripts/travis.py fold_start default "Testing default backend"
+	GEVENTTEST_COVERAGE=1 make alltest
+	@${PYTHON} scripts/travis.py fold_end default
 	GEVENTTEST_COVERAGE=1 make cffibackendtest
 # because we set parallel=true, each run produces new and different coverage files; they all need
 # to be combined
@@ -92,17 +89,17 @@ allbackendtest:
 
 
 cffibackendtest:
-	${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
-	GEVENT_CORE_CFFI_ONLY=libuv GEVENTTEST_COVERAGE=1 make alltest
-	${PYTHON} scripts/travis.py fold_end libuv
-	${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
-	GEVENT_CORE_CFFI_ONLY=libev make alltest
-	${PYTHON} scripts/travis.py fold_end libev
+	@${PYTHON} scripts/travis.py fold_start libuv "Testing libuv backend"
+	GEVENT_LOOP=libuv GEVENTTEST_COVERAGE=1 make alltest
+	@${PYTHON} scripts/travis.py fold_end libuv
+	@${PYTHON} scripts/travis.py fold_start libev "Testing libev CFFI backend"
+	GEVENT_LOOP=libev-cffi make alltest
+	@${PYTHON} scripts/travis.py fold_end libev
 
 leaktest: test_prelim
-	${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
+	@${PYTHON} scripts/travis.py fold_start leaktest "Running leak tests"
 	cd src/greentest && GEVENT_RESOLVER=thread GEVENTTEST_LEAKCHECK=1 ${PYTHON} testrunner.py --config known_failures.py --quiet --ignore tests_that_dont_do_leakchecks.txt
-	${PYTHON} scripts/travis.py fold_end leaktest
+	@${PYTHON} scripts/travis.py fold_end leaktest
 
 bench:
 	${PYTHON} src/greentest/bench_sendall.py
@@ -166,8 +163,8 @@ $(PYPY3):
 
 
 develop:
-	${PYTHON} scripts/travis.py fold_start install "Installing gevent"
-	echo python is at `which $(PYTHON)`
+	@${PYTHON} scripts/travis.py fold_start install "Installing gevent"
+	@echo python is at `which $(PYTHON)`
 # First install a newer pip so that it can use the wheel cache
 # (only needed until travis upgrades pip to 7.x; note that the 3.5
 # environment uses pip 7.1 by default)
@@ -176,7 +173,7 @@ develop:
 # disables the cache.
 # We need wheel>=0.26 on Python 3.5. See previous revisions.
 	GEVENTSETUP_EV_VERIFY=3 ${PYTHON} -m pip install -U -r dev-requirements.txt
-	${PYTHON} scripts/travis.py fold_end install
+	@${PYTHON} scripts/travis.py fold_end install
 
 test-py27: $(PY27)
 	PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop lint leaktest allbackendtest
@@ -200,7 +197,13 @@ test-pypy3: $(PYPY3)
 	PYTHON=$(PYPY3) PATH=$(BUILD_RUNTIMES)/versions/pypy3.5_5101/bin:$(PATH) make develop basictest
 
 test-py27-noembed: $(PY27)
+	@python2.7.14 scripts/travis.py fold_start conf_libev "Configuring libev"
 	cd deps/libev && ./configure --disable-dependency-tracking && make
+	@python2.7.14 scripts/travis.py fold_end conf_libev
+	@python2.7.14 scripts/travis.py fold_start conf_cares "Configuring cares"
 	cd deps/c-ares && ./configure --disable-dependency-tracking && make
+	@python2.7.14 scripts/travis.py fold_end conf_cares
+	@python2.7.14 scripts/travis.py fold_start conf_libuv "Configuring libuv"
 	cd deps/libuv && ./autogen.sh && ./configure --disable-static && make
-	CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest
+	@python2.7.14 scripts/travis.py fold_end conf_libuv
+	CPPFLAGS="-Ideps/libev -Ideps/c-ares -Ideps/libuv/include" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs -Ldeps/libuv/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs:$(PWD)/deps/libuv/.libs" EMBED=0 GEVENT_LOOP=libev-cext PYTHON=python2.7.14 PATH=$(BUILD_RUNTIMES)/versions/python2.7.14/bin:$(PATH) make develop basictest

appveyor.yml

@@ -26,7 +26,7 @@ environment:
       PYTHON_VERSION: "3.6.x" # currently 3.6.0
       PYTHON_ARCH: "64"
       PYTHON_EXE: python
-      GEVENT_CORE_CFFI_ONLY: libuv
+      GEVENT_LOOP: libuv
 
     - PYTHON: "C:\\Python27-x64"
       PYTHON_VERSION: "2.7.x" # currently 2.7.13

doc/conf.py

@@ -87,7 +87,7 @@ release = version
 exclude_trees = ['_build']
 
 # The reST default role (used for this markup: `text`) to use for all documents.
-#default_role = None
+default_role = 'obj'
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 add_function_parentheses = True

doc/gevent.rst

@@ -191,3 +191,9 @@ Timeouts
     :special-members: __enter__, __exit__
 
 .. autofunction:: with_timeout
+
+
+Configuration
+=============
+
+.. autoclass:: gevent._config.Config

src/gevent/__init__.py

@@ -4,6 +4,9 @@ gevent is a coroutine-based Python networking library that uses greenlet
 to provide a high-level synchronous API on top of libev event loop.
 
 See http://www.gevent.org/ for the documentation.
+
+.. versionchanged:: 1.3a2
+   Add the `config` object.
 """
 
 from __future__ import absolute_import
@@ -43,6 +46,8 @@ __all__ = [
     'reinit',
     'getswitchinterval',
     'setswitchinterval',
+    # Added in 1.3a2
+    'config',
 ]
 
 
@@ -72,11 +77,14 @@ except AttributeError:
             global _switchinterval
             _switchinterval = interval
 
+from gevent._config import config
 from gevent.hub import get_hub, iwait, wait
 from gevent.greenlet import Greenlet, joinall, killall
 joinall = joinall # export for pylint
 spawn = Greenlet.spawn
 spawn_later = Greenlet.spawn_later
+#: The singleton configuration object for gevent.
+config = config
 
 from gevent.timeout import Timeout, with_timeout
 from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit

src/gevent/_config.py

+# Copyright (c) 2018 gevent. See LICENSE for details.
+"""
+gevent tunables.
+
+This should be used as ``from gevent import config``. That variable
+is an object of :class:`Config`.
+
+.. versionadded:: 1.3a2
+"""
+
+from __future__ import print_function, absolute_import, division
+
+import importlib
+import os
+import sys
+import textwrap
+
+from gevent._compat import string_types
+
+__all__ = [
+    'config',
+]
+
+ALL_SETTINGS = []
+
+class SettingType(type):
+    # pylint:disable=bad-mcs-classmethod-argument
+
+    def __new__(cls, name, bases, cls_dict):
+        if name == 'Setting':
+            return type.__new__(cls, name, bases, cls_dict)
+
+        cls_dict["order"] = len(ALL_SETTINGS)
+        if 'name' not in cls_dict:
+            cls_dict['name'] = name.lower()
+
+        if 'environment_key' not in cls_dict:
+            cls_dict['environment_key'] = 'GEVENT_' + cls_dict['name'].upper()
+
+
+        new_class = type.__new__(cls, name, bases, cls_dict)
+        new_class.fmt_desc(cls_dict.get("desc", ""))
+        new_class.__doc__ = new_class.desc
+        ALL_SETTINGS.append(new_class)
+
+        if new_class.document:
+            setting_name = cls_dict['name']
+
+            def getter(self):
+                return self.settings[setting_name].get()
+
+            def setter(self, value): # pragma: no cover
+                # The setter should never be hit, Config has a
+                # __setattr__ that would override. But for the sake
+                # of consistency we provide one.
+                self.settings[setting_name].set(value)
+
+            prop = property(getter, setter, doc=new_class.__doc__)
+
+            setattr(Config, cls_dict['name'], prop)
+        return new_class
+
+    def fmt_desc(cls, desc):
+        desc = textwrap.dedent(desc).strip()
+        if hasattr(cls, 'shortname_map'):
+            desc += (
+                "\n\nThis is an importable value. It can be "
+                "given as a string naming an importable object, "
+                "or a list of strings in preference order and the first "
+                "successfully importable object will be used. (Separate values "
+                "in the environment variable with commas.) "
+                "It can also be given as the callable object itself (in code). "
+            )
+            if cls.shortname_map:
+                desc += "Shorthand names for default objects are %r" % (list(cls.shortname_map),)
+        if getattr(cls.validate, '__doc__'):
+            desc += '\n\n' + textwrap.dedent(cls.validate.__doc__).strip()
+        if isinstance(cls.default, str) and hasattr(cls, 'shortname_map'):
+            default = "`%s`" % (cls.default,)
+        else:
+            default = "`%r`" % (cls.default,)
+        desc += "\n\nThe default value is %s" % (default,)
+        desc += ("\n\nThe environment variable ``%s`` "
+                 "can be used to control this." % (cls.environment_key,))
+        setattr(cls, "desc", desc)
+        return desc
+
+def validate_invalid(value):
+    raise ValueError("Not a valid value: %r" % (value,))
+
+def validate_bool(value):
+    """
+    This is a boolean value.
+
+    In the environment variable, it may be given as ``1``, ``true``,
+    ``on`` or ``yes`` for `True`, or ``0``, ``false``, ``off``, or
+    ``no`` for `False`.
+    """
+    if isinstance(value, string_types):
+        value = value.lower().strip()
+        if value in ('1', 'true', 'on', 'yes'):
+            value = True
+        elif value in ('0', 'false', 'off', 'no') or not value:
+            value = False
+        else:
+            raise ValueError("Invalid boolean string: %r" % (value,))
+    return bool(value)
+
+def validate_anything(value):
+    return value
+
+class Setting(object):
+    name = None
+    value = None
+    validate = staticmethod(validate_invalid)
+    default = None
+    environment_key = None
+    document = True
+
+    desc = """\
+
+    A long ReST description.
+
+    The first line should be a single sentence.
+
+    """
+
+    def _convert(self, value):
+        if isinstance(value, string_types):
+            return value.split(',')
+        return value
+
+    def _default(self):
+        result = os.environ.get(self.environment_key, self.default)
+        result = self._convert(result)
+        return result
+
+    def get(self):
+        # If we've been specifically set, return it
+        if 'value' in self.__dict__:
+            return self.value
+        # Otherwise, read from the environment and reify
+        # so we return consistent results.
+        self.value = self.validate(self._default())
+        return self.value
+
+    def set(self, val):
+        self.value = self.validate(self._convert(val))
+
+
+Setting = SettingType('Setting', (Setting,), dict(Setting.__dict__))
+
+def make_settings():
+    """
+    Return fresh instances of all classes defined in `ALL_SETTINGS`.
+    """
+    settings = {}
+    for setting_kind in ALL_SETTINGS:
+        setting = setting_kind()
+        assert setting.name not in settings
+        settings[setting.name] = setting
+    return settings
+
+
+class Config(object):
+    """
+    Global configuration for gevent.
+
+    There is one instance of this object at ``gevent.config``. If you
+    are going to make changes in code, instead of using the documented
+    environment variables, you need to make the changes before using
+    any parts of gevent that might need those settings. For example::
+
+        >>> from gevent import config
+        >>> config.fileobject = 'thread'
+
+        >>> from gevent import fileobject
+        >>> fileobject.FileObject.__name__
+        'FileObjectThread'
+
+    .. versionadded:: 1.3a2
+
+    """
+
+    def __init__(self):
+        self.settings = make_settings()
+
+    def __getattr__(self, name):
+        if name not in self.settings:
+            raise AttributeError("No configuration setting for: %r" % name)
+        return self.settings[name].get()
+
+    def __setattr__(self, name, value):
+        if name != "settings" and name in self.settings:
+            self.set(name, value)
+        else:
+            super(Config, self).__setattr__(name, value)
+
+    def set(self, name, value):
+        if name not in self.settings:
+            raise AttributeError("No configuration setting for: %r" % name)
+        self.settings[name].set(value)
+
+    def __dir__(self):
+        return list(self.settings)
+
+
+class ImportableSetting(object):
+
+    def _import(self, path, _NONE=object):
+        # pylint:disable=too-many-branches
+        if isinstance(path, list):
+            if not path:
+                raise ImportError('Cannot import from empty list: %r' % (path, ))
+
+            for item in path[:-1]:
+                try:
+                    return self._import(item)
+                except ImportError:
+                    pass
+
+            return self._import(path[-1])
+
+        if not isinstance(path, string_types):
+            return path
+
+        if '.' not in path:
+            raise ImportError("Cannot import %r. "
+                              "Required format: [path/][package.]module.class. "
+                              "Or choice from %r"
+                              % (path, list(self.shortname_map)))
+
+        if '/' in path:
+            # This is dangerous, subject to race conditions, and
+            # may not work properly for things like namespace packages
+            import warnings
+            warnings.warn("Absolute paths are deprecated and will be removed in 1.4."
+                          "Please put the package on sys.path first",
+                          DeprecationWarning)
+            package_path, path = path.rsplit('/', 1)
+            sys.path = [package_path] + sys.path
+        else:
+            package_path = None
+
+        try:
+            module, item = path.rsplit('.', 1)
+            module = importlib.import_module(module)
+            x = getattr(module, item, _NONE)
+            if x is _NONE:
+                raise ImportError('Cannot import %r from %r' % (item, module))
+            return x
+        finally:
+            if package_path:
+                try:
+                    sys.path.remove(package_path)
+                except ValueError: # pragma: no cover
+                    pass
+
+    shortname_map = {}
+
+    def validate(self, value):
+        if isinstance(value, type):
+            return value
+        return self._import([self.shortname_map.get(x, x) for x in value])
+
+
+class Resolver(ImportableSetting, Setting):
+
+    desc = """\
+    The callable that will be used to create
+    :attr:`gevent.hub.Hub.resolver`.
+
+    See :doc:`dns` for more information.
+    """
+
+    default = [
+        'thread',
+        'dnspython',
+        'ares',
+        'block',
+    ]
+
+    shortname_map = {
+        'ares': 'gevent.resolver.ares.Resolver',
+        'thread': 'gevent.resolver.thread.Resolver',
+        'block': 'gevent.resolver.blocking.Resolver',
+        'dnspython': 'gevent.resolver.dnspython.Resolver',
+    }
+
+
+
+class Threadpool(ImportableSetting, Setting):
+
+    desc = """\
+    The kind of threadpool we use.
+    """
+
+    default = 'gevent.threadpool.ThreadPool'
+
+
+class Loop(ImportableSetting, Setting):
+
+    desc = """\
+    The kind of the loop we use.
+
+    """
+
+    default = [
+        'libev-cext',
+        'libev-cffi',
+        'libuv-cffi',
+    ]
+
+    shortname_map = {
+        'libev-cext': 'gevent.libev.corecext.loop',
+        'libev-cffi': 'gevent.libev.corecffi.loop',
+        'libuv-cffi': 'gevent.libuv.loop.loop',
+    }
+
+    shortname_map['libuv'] = shortname_map['libuv-cffi']
+
+
+class FormatContext(ImportableSetting, Setting):
+    name = 'format_context'
+
+    # using pprint.pformat can override custom __repr__ methods on dict/list
+    # subclasses, which can be a security concern
+    default = 'pprint.saferepr'
+
+
+class LibevBackend(Setting):
+    name = 'libev_backend'
+    environment_key = 'GEVENT_BACKEND'
+
+    desc = """\
+    The backend for libev, such as 'select'
+    """
+
+    default = None
+
+    validate = staticmethod(validate_anything)
+
+
+class FileObject(ImportableSetting, Setting):
+    desc = """\
+    The kind of ``FileObject`` we will use.
+
+    See :mod:`gevent.fileobject` for a detailed description.
+
+    """
+    environment_key = 'GEVENT_FILE'
+
+    default = [
+        'posix',
+        'thread',
+    ]
+
+    shortname_map = {
+        'thread': 'gevent._fileobjectcommon.FileObjectThread',
+        'posix': 'gevent._fileobjectposix.FileObjectPosix',
+        'block': 'gevent._fileobjectcommon.FileObjectBlock'
+    }
+
+
+class WatchChildren(Setting):
+    desc = """\
+    Should we *not* watch children with the event loop watchers?
+
+    This is an advanced setting.
+
+    See :mod:`gevent.os` for a detailed description.
+    """
+    name = 'disable_watch_children'
+    environment_key = 'GEVENT_NOWAITPID'
+    default = False
+
+    validate = staticmethod(validate_bool)
+
+
+class TraceMalloc(Setting):
+    name = 'trace_malloc'
+    environment_key = 'PYTHONTRACEMALLOC'
+    default = False
+    validate = staticmethod(validate_bool)
+
+    desc = """\
+    Should FFI objects track their allocation?
+
+    This is only useful for low-level debugging.
+
+    On Python 3, this environment variable is built in to the
+    interpreter, and it may also be set with the ``-X
+    tracemalloc`` command line argument.
+
+    On Python 2, gevent interprets this argument and adds extra
+    tracking information for FFI objects.
+    """
+
+# The ares settings are all interpreted by
+# gevent/resolver/ares.pyx, so we don't do
+# any validation here.
+
+class AresSettingMixin(object):
+
+    document = False
+
+    @property
+    def kwarg_name(self):
+        return self.name[5:]
+
+    validate = staticmethod(validate_anything)
+
+    _convert = staticmethod(validate_anything)
+
+class AresFlags(AresSettingMixin, Setting):
+    name = 'ares_flags'
+    default = None
+    environment_key = 'GEVENTARES_FLAGS'
+
+class AresTimeout(AresSettingMixin, Setting):
+    name = 'ares_timeout'
+    default = None
+    environment_key = 'GEVENTARES_TIMEOUT'
+
+class AresTries(AresSettingMixin, Setting):
+    name = 'ares_tries'
+    default = None
+    environment_key = 'GEVENTARES_TRIES'
+
+class AresNdots(AresSettingMixin, Setting):
+    name = 'ares_ndots'
+    default = None
+    environment_key = 'GEVENTARES_NDOTS'
+
+class AresUDPPort(AresSettingMixin, Setting):
+    name = 'ares_udp_port'
+    default = None
+    environment_key = 'GEVENTARES_UDP_PORT'
+
+class AresTCPPort(AresSettingMixin, Setting):
+    name = 'ares_tcp_port'
+    default = None
+    environment_key = 'GEVENTARES_TCP_PORT'
+
+class AresServers(AresSettingMixin, Setting):
+    document = True
+    name = 'ares_servers'
+    default = None
+    environment_key = 'GEVENTARES_SERVERS'
+
+
+config = Config()
+
+# Go ahead and attempt to import the loop when this class is
+# instantiated. The hub won't work if the loop can't be found. This
+# can solve problems with the class being imported from multiple
+# threads at once, leading to one of the imports failing.
+# factories are themselves handled lazily. See #687.
+
+# Don't cache it though, in case the user re-configures through the
+# API.
+
+try:
+    Loop().get()
+except ImportError: # pragma: no cover
+    pass

src/gevent/_ffi/watcher.py

@@ -10,14 +10,17 @@ import signal as signalmodule
 import functools
 import warnings
 
+from gevent._config import config
+
 try:
     from tracemalloc import get_object_traceback
 
     def tracemalloc(init):
-        # PYTHONTRACEMALLOC env var controls this.
+        # PYTHONTRACEMALLOC env var controls this on Python 3.
         return init
 except ImportError: # Python < 3.4
-    if os.getenv('PYTHONTRACEMALLOC'):
+
+    if config.trace_malloc:
         # Use the same env var to turn this on for Python 2
         import traceback
 

src/gevent/_fileobjectcommon.py

+from __future__ import absolute_import, print_function, division
 
 try:
     from errno import EBADF
 except ImportError:
     EBADF = 9
 
+import os
 from io import TextIOWrapper
+import functools
+import sys
+
+
+from gevent.hub import get_hub
+from gevent._compat import integer_types
+from gevent._compat import reraise
+from gevent.lock import Semaphore, DummySemaphore
 
 class cancel_wait_ex(IOError):
 
@@ -136,3 +146,130 @@ class FileObjectBase(object):
 
     def __exit__(self, *args):
         self.close()
+
+class FileObjectBlock(FileObjectBase):
+
+    def __init__(self, fobj, *args, **kwargs):
+        closefd = kwargs.pop('close', True)
+        if kwargs:
+            raise TypeError('Unexpected arguments: %r' % kwargs.keys())
+        if isinstance(fobj, integer_types):
+            if not closefd:
+                # we cannot do this, since fdopen object will close the descriptor
+                raise TypeError('FileObjectBlock does not support close=False on an fd.')
+            fobj = os.fdopen(fobj, *args)
+        super(FileObjectBlock, self).__init__(fobj, closefd)
+
+    def _do_close(self, fobj, closefd):
+        fobj.close()
+
+class FileObjectThread(FileObjectBase):
+    """
+    A file-like object wrapping another file-like object, performing all blocking
+    operations on that object in a background thread.
+
+    .. caution::
+        Attempting to change the threadpool or lock of an existing FileObjectThread
+        has undefined consequences.
+
+    .. versionchanged:: 1.1b1
+       The file object is closed using the threadpool. Note that whether or
+       not this action is synchronous or asynchronous is not documented.
+
+    """
+
+    def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
+        """
+        :param fobj: The underlying file-like object to wrap, or an integer fileno
+           that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
+        :keyword bool lock: If True (the default) then all operations will
+           be performed one-by-one. Note that this does not guarantee that, if using
+           this file object from multiple threads/greenlets, operations will be performed
+           in any particular order, only that no two operations will be attempted at the
+           same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
+           file operations with an external resource.
+        :keyword bool close: If True (the default) then when this object is closed,
+           the underlying object is closed as well.
+        """
+        closefd = close
+        self.threadpool = threadpool or get_hub().threadpool
+        self.lock = lock
+        if self.lock is True:
+            self.lock = Semaphore()
+        elif not self.lock:
+            self.lock = DummySemaphore()
+        if not hasattr(self.lock, '__enter__'):
+            raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
+        if isinstance(fobj, integer_types):
+            if not closefd:
+                # we cannot do this, since fdopen object will close the descriptor
+                raise TypeError('FileObjectThread does not support close=False on an fd.')
+            if mode is None:
+                assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
+                fobj = os.fdopen(fobj)
+            else:
+                fobj = os.fdopen(fobj, mode, bufsize)
+
+        self.__io_holder = [fobj] # signal for _wrap_method
+        super(FileObjectThread, self).__init__(fobj, closefd)
+
+    def _do_close(self, fobj, closefd):
+        self.__io_holder[0] = None # for _wrap_method
+        try:
+            with self.lock:
+                self.threadpool.apply(fobj.flush)
+        finally:
+            if closefd:
+                # Note that we're not taking the lock; older code
+                # did fobj.close() without going through the threadpool at all,
+                # so acquiring the lock could potentially introduce deadlocks
+                # that weren't present before. Avoiding the lock doesn't make
+                # the existing race condition any worse.
+                # We wrap the close in an exception handler and re-raise directly
+                # to avoid the (common, expected) IOError from being logged by the pool
+                def close():
+                    try:
+                        fobj.close()
+                    except: # pylint:disable=bare-except
+                        return sys.exc_info()
+                exc_info = self.threadpool.apply(close)
+                if exc_info:
+                    reraise(*exc_info)
+
+    def _do_delegate_methods(self):
+        super(FileObjectThread, self)._do_delegate_methods()
+        if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
+            self.read1 = self.read
+        self.__io_holder[0] = self._io
+
+    def _extra_repr(self):
+        return ' threadpool=%r' % (self.threadpool,)
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        line = self.readline()
+        if line:
+            return line
+        raise StopIteration
+    __next__ = next
+
+    def _wrap_method(self, method):
+        # NOTE: We are careful to avoid introducing a refcycle
+        # within self. Our wrapper cannot refer to self.
+        io_holder = self.__io_holder
+        lock = self.lock
+        threadpool = self.threadpool
+
+        @functools.wraps(method)
+        def thread_method(*args, **kwargs):
+            if io_holder[0] is None:
+                # This is different than FileObjectPosix, etc,
+                # because we want to save the expensive trip through
+                # the threadpool.
+                raise FileObjectClosed()
+            with lock:
+                return threadpool.apply(method, args, kwargs)
+
+        return thread_method

src/gevent/core.py

 # Copyright (c) 2009-2015 Denis Bilenko and gevent contributors. See LICENSE for details.
+"""
+Deprecated; this does not reflect all the possible options
+and its interface varies.
+
+.. versionchanged:: 1.3a2
+    Deprecated.
+"""
 from __future__ import absolute_import
 
-import os
+import sys
 
+from gevent._config import config
 from gevent._util import copy_globals
 
-try:
-    if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
-        raise ImportError("Not attempting corecext")
-
-    from gevent.libev import corecext as _core
-except ImportError:
-    if os.environ.get('GEVENT_CORE_CEXT_ONLY'):
-        raise
-
-    # CFFI/PyPy
-    lib = os.environ.get('GEVENT_CORE_CFFI_ONLY')
-    if lib == 'libuv':
-        from gevent.libuv import loop as _core
-    else:
-        try:
-            from gevent.libev import corecffi as _core
-        except ImportError:
-            from gevent.libuv import loop as _core
+_core = sys.modules[config.loop.__module__]
 
 copy_globals(_core, globals())
 

src/gevent/fileobject.py

@@ -35,31 +35,12 @@ Classes
 """
 from __future__ import absolute_import
 
-import functools
-import sys
-import os
-
-from gevent._fileobjectcommon import FileObjectClosed
-from gevent._fileobjectcommon import FileObjectBase
-from gevent.hub import get_hub
-from gevent._compat import integer_types
-from gevent._compat import reraise
-from gevent.lock import Semaphore, DummySemaphore
-
-
-PYPY = hasattr(sys, 'pypy_version_info')
-
-if hasattr(sys, 'exc_clear'):
-    def _exc_clear():
-        sys.exc_clear()
-else:
-    def _exc_clear():
-        return
-
+from gevent._config import config
 
 __all__ = [
     'FileObjectPosix',
     'FileObjectThread',
+    'FileObjectBlock',
     'FileObject',
 ]
 
@@ -71,149 +52,10 @@ else:
     del fcntl
     from gevent._fileobjectposix import FileObjectPosix
 
+from gevent._fileobjectcommon import FileObjectThread
+from gevent._fileobjectcommon import FileObjectBlock
 
-class FileObjectThread(FileObjectBase):
-    """
-    A file-like object wrapping another file-like object, performing all blocking
-    operations on that object in a background thread.
-
-    .. caution::
-        Attempting to change the threadpool or lock of an existing FileObjectThread
-        has undefined consequences.
-
-    .. versionchanged:: 1.1b1
-       The file object is closed using the threadpool. Note that whether or
-       not this action is synchronous or asynchronous is not documented.
-
-    """
-
-    def __init__(self, fobj, mode=None, bufsize=-1, close=True, threadpool=None, lock=True):
-        """
-        :param fobj: The underlying file-like object to wrap, or an integer fileno
-           that will be pass to :func:`os.fdopen` along with *mode* and *bufsize*.
-        :keyword bool lock: If True (the default) then all operations will
-           be performed one-by-one. Note that this does not guarantee that, if using
-           this file object from multiple threads/greenlets, operations will be performed
-           in any particular order, only that no two operations will be attempted at the
-           same time. You can also pass your own :class:`gevent.lock.Semaphore` to synchronize
-           file operations with an external resource.
-        :keyword bool close: If True (the default) then when this object is closed,
-           the underlying object is closed as well.
-        """
-        closefd = close
-        self.threadpool = threadpool or get_hub().threadpool
-        self.lock = lock
-        if self.lock is True:
-            self.lock = Semaphore()
-        elif not self.lock:
-            self.lock = DummySemaphore()
-        if not hasattr(self.lock, '__enter__'):
-            raise TypeError('Expected a Semaphore or boolean, got %r' % type(self.lock))
-        if isinstance(fobj, integer_types):
-            if not closefd:
-                # we cannot do this, since fdopen object will close the descriptor
-                raise TypeError('FileObjectThread does not support close=False on an fd.')
-            if mode is None:
-                assert bufsize == -1, "If you use the default mode, you can't choose a bufsize"
-                fobj = os.fdopen(fobj)
-            else:
-                fobj = os.fdopen(fobj, mode, bufsize)
-
-        self.__io_holder = [fobj] # signal for _wrap_method
-        super(FileObjectThread, self).__init__(fobj, closefd)
-
-    def _do_close(self, fobj, closefd):
-        self.__io_holder[0] = None # for _wrap_method
-        try:
-            with self.lock:
-                self.threadpool.apply(fobj.flush)
-        finally:
-            if closefd:
-                # Note that we're not taking the lock; older code
-                # did fobj.close() without going through the threadpool at all,
-                # so acquiring the lock could potentially introduce deadlocks
-                # that weren't present before. Avoiding the lock doesn't make
-                # the existing race condition any worse.
-                # We wrap the close in an exception handler and re-raise directly
-                # to avoid the (common, expected) IOError from being logged by the pool
-                def close():
-                    try:
-                        fobj.close()
-                    except: # pylint:disable=bare-except
-                        return sys.exc_info()
-                exc_info = self.threadpool.apply(close)
-                if exc_info:
-                    reraise(*exc_info)
-
-    def _do_delegate_methods(self):
-        super(FileObjectThread, self)._do_delegate_methods()
-        if not hasattr(self, 'read1') and 'r' in getattr(self._io, 'mode', ''):
-            self.read1 = self.read
-        self.__io_holder[0] = self._io
-
-    def _extra_repr(self):
-        return ' threadpool=%r' % (self.threadpool,)
-
-    def __iter__(self):
-        return self
-
-    def next(self):
-        line = self.readline()
-        if line:
-            return line
-        raise StopIteration
-    __next__ = next
-
-    def _wrap_method(self, method):
-        # NOTE: We are careful to avoid introducing a refcycle
-        # within self. Our wrapper cannot refer to self.
-        io_holder = self.__io_holder
-        lock = self.lock
-        threadpool = self.threadpool
-
-        @functools.wraps(method)
-        def thread_method(*args, **kwargs):
-            if io_holder[0] is None:
-                # This is different than FileObjectPosix, etc,
-                # because we want to save the expensive trip through
-                # the threadpool.
-                raise FileObjectClosed()
-            with lock:
-                return threadpool.apply(method, args, kwargs)
-
-        return thread_method
-
-
-try:
-    FileObject = FileObjectPosix
-except NameError:
-    FileObject = FileObjectThread
-
-
-class FileObjectBlock(FileObjectBase):
-
-    def __init__(self, fobj, *args, **kwargs):
-        closefd = kwargs.pop('close', True)
-        if kwargs:
-            raise TypeError('Unexpected arguments: %r' % kwargs.keys())
-        if isinstance(fobj, integer_types):
-            if not closefd:
-                # we cannot do this, since fdopen object will close the descriptor
-                raise TypeError('FileObjectBlock does not support close=False on an fd.')
-            fobj = os.fdopen(fobj, *args)
-        super(FileObjectBlock, self).__init__(fobj, closefd)
-
-    def _do_close(self, fobj, closefd):
-        fobj.close()
 
-config = os.environ.get('GEVENT_FILE')
-if config:
-    klass = {'thread': 'gevent.fileobject.FileObjectThread',
-             'posix': 'gevent.fileobject.FileObjectPosix',
-             'block': 'gevent.fileobject.FileObjectBlock'}.get(config, config)
-    if klass.startswith('gevent.fileobject.'):
-        FileObject = globals()[klass.split('.', 2)[-1]]
-    else:
-        from gevent.hub import _import
-        FileObject = _import(klass)
-    del klass
+# None of the possible objects can live in this module because
+# we would get an import cycle and the config couldn't be set from code.
+FileObject = config.fileobject

src/gevent/hub.py

@@ -26,6 +26,7 @@ __all__ = [
     'Waiter',
 ]
 
+from gevent._config import config
 from gevent._compat import string_types
 from gevent._compat import xrange
 from gevent._util import _NONE
@@ -389,89 +390,29 @@ def set_hub(hub):
     _threadlocal.hub = hub
 
 
-def _import(path):
-    # pylint:disable=too-many-branches
-    if isinstance(path, list):
-        if not path:
-            raise ImportError('Cannot import from empty list: %r' % (path, ))
 
-        for item in path[:-1]:
-            try:
-                return _import(item)
-            except ImportError:
-                pass
-
-        return _import(path[-1])
-
-    if not isinstance(path, string_types):
-        return path
-
-    if '.' not in path:
-        raise ImportError("Cannot import %r (required format: [path/][package.]module.class)" % path)
-
-    if '/' in path:
-        # This is dangerous, subject to race conditions, and
-        # may not work properly for things like namespace packages
-        import warnings
-        warnings.warn("Absolute paths are deprecated. Please put the package "
-                      "on sys.path first",
-                      DeprecationWarning)
-        package_path, path = path.rsplit('/', 1)
-        sys.path = [package_path] + sys.path
-    else:
-        package_path = None
-
-    try:
-        module, item = path.rsplit('.', 1)
-        x = __import__(module)
-        for attr in path.split('.')[1:]:
-            oldx = x
-            x = getattr(x, attr, _NONE)
-            if x is _NONE:
-                raise ImportError('Cannot import %r from %r' % (attr, oldx))
-        return x
-    finally:
-        if '/' in path:
-            try:
-                sys.path.remove(package_path)
-            except ValueError:
-                pass
 
-
-def config(default, envvar):
+def _config(default, envvar):
     result = os.environ.get(envvar) or default # absolute import gets confused pylint: disable=no-member
     if isinstance(result, string_types):
         return result.split(',')
     return result
 
 
-def resolver_config(default, envvar):
-    result = config(default, envvar)
-    return [_resolvers.get(x, x) for x in result]
-
-
-_resolvers = {
-    'ares': 'gevent.resolver.ares.Resolver',
-    'thread': 'gevent.resolver.thread.Resolver',
-    'block': 'gevent.resolver.blocking.Resolver',
-    'dnspython': 'gevent.resolver.dnspython.Resolver',
-}
-
-
-_DEFAULT_LOOP_CLASS = 'gevent.core.loop'
-
 
 class Hub(RawGreenlet):
-    """A greenlet that runs the event loop.
+    """
+    A greenlet that runs the event loop.
 
     It is created automatically by :func:`get_hub`.
 
-    **Switching**
+    .. rubric:: Switching
 
-    Every time this greenlet (i.e., the event loop) is switched *to*, if
-    the current greenlet has a ``switch_out`` method, it will be called. This
-    allows a greenlet to take some cleanup actions before yielding control. This method
-    should not call any gevent blocking functions.
+    Every time this greenlet (i.e., the event loop) is switched *to*,
+    if the current greenlet has a ``switch_out`` method, it will be
+    called. This allows a greenlet to take some cleanup actions before
+    yielding control. This method should not call any gevent blocking
+    functions.
     """
 
     #: If instances of these classes are raised into the event loop,
@@ -483,37 +424,8 @@ class Hub(RawGreenlet):
     #: do not get logged/printed when raised by the event loop.
     NOT_ERROR = (GreenletExit, SystemExit)
 
-    loop_class = config(_DEFAULT_LOOP_CLASS, 'GEVENT_LOOP')
-    # For the standard class, go ahead and import it when this class
-    # is defined. This is no loss of generality because the envvar is
-    # only read when this class is defined, and we know that the
-    # standard class will be available. This can solve problems with
-    # the class being imported from multiple threads at once, leading
-    # to one of the imports failing. Only do this for the object we
-    # need in the constructor, as the rest of the factories are
-    # themselves handled lazily. See #687. (People using a custom loop_class
-    # can probably manage to get_hub() from the main thread or otherwise import
-    # that loop_class themselves.)
-    if loop_class == [_DEFAULT_LOOP_CLASS]:
-        loop_class = [_import(loop_class)]
-
-    resolver_class = [
-        'gevent.resolver.thread.Resolver',
-        'gevent.resolver.dnspython.Resolver',
-        'gevent.resolver.ares.Resolver',
-        'gevent.resolver.blacking.Resolver',
-    ]
-    #: The class or callable object, or the name of a factory function or class,
-    #: that will be used to create :attr:`resolver`. By default, configured according to
-    #: :doc:`dns`. If a list, a list of objects in preference order.
-    resolver_class = resolver_config(resolver_class, 'GEVENT_RESOLVER')
-    threadpool_class = config('gevent.threadpool.ThreadPool', 'GEVENT_THREADPOOL')
-    backend = config(None, 'GEVENT_BACKEND')
-    threadpool_size = 10
 
-    # using pprint.pformat can override custom __repr__ methods on dict/list
-    # subclasses, which can be a security concern
-    format_context = 'pprint.saferepr'
+    threadpool_size = 10
 
 
     def __init__(self, loop=None, default=None):
@@ -530,13 +442,21 @@ class Hub(RawGreenlet):
         else:
             if default is None and get_ident() != MAIN_THREAD:
                 default = False
-            loop_class = _import(self.loop_class)
+
             if loop is None:
                 loop = self.backend
-            self.loop = loop_class(flags=loop, default=default)
+            self.loop = self.loop_class(flags=loop, default=default) # pylint:disable=not-callable
         self._resolver = None
         self._threadpool = None
-        self.format_context = _import(self.format_context)
+        self.format_context = config.format_context
+
+    @property
+    def loop_class(self):
+        return config.loop
+
+    @property
+    def backend(self):
+        return config.libev_backend
 
     def __repr__(self):
         if self.loop is None:
@@ -787,11 +707,13 @@ class Hub(RawGreenlet):
         if _threadlocal.hub is self:
             _threadlocal.hub = None
 
+    @property
+    def resolver_class(self):
+        return config.resolver
+
     def _get_resolver(self):
         if self._resolver is None:
-            if self.resolver_class is not None:
-                self.resolver_class = _import(self.resolver_class)
-                self._resolver = self.resolver_class(hub=self)
+            self._resolver = self.resolver_class(hub=self) # pylint:disable=not-callable
         return self._resolver
 
     def _set_resolver(self, value):
@@ -802,10 +724,14 @@ class Hub(RawGreenlet):
 
     resolver = property(_get_resolver, _set_resolver, _del_resolver)
 
+
+    @property
+    def threadpool_class(self):
+        return config.threadpool
+
     def _get_threadpool(self):
         if self._threadpool is None:
-            if self.threadpool_class is not None:
-                self.threadpool_class = _import(self.threadpool_class)
+            # pylint:disable=not-callable
             self._threadpool = self.threadpool_class(self.threadpool_size, hub=self)
         return self._threadpool
 

src/gevent/os.py

@@ -46,6 +46,7 @@ from __future__ import absolute_import
 import os
 import sys
 from gevent.hub import get_hub, reinit
+from gevent._config import config
 from gevent._compat import PY3
 from gevent._util import copy_globals
 import errno
@@ -418,7 +419,7 @@ if hasattr(os, 'fork'):
             __extensions__.append('forkpty_and_watch')
 
         # Watch children by default
-        if not os.getenv('GEVENT_NOWAITPID'):
+        if not config.disable_watch_children:
             # Broken out into separate functions instead of simple name aliases
             # for documentation purposes.
             def fork(*args, **kwargs):

src/gevent/resolver/ares.py

@@ -27,6 +27,9 @@ from gevent.socket import SOCK_DGRAM
 from gevent.socket import SOCK_RAW
 from gevent.socket import AI_NUMERICHOST
 
+from gevent._config import config
+from gevent._config import AresSettingMixin
+
 from .cares import channel, InvalidIP # pylint:disable=import-error,no-name-in-module
 from . import _lookup_port as lookup_port
 from . import _resolve_special
@@ -84,12 +87,11 @@ class Resolver(AbstractResolver):
             hub = get_hub()
         self.hub = hub
         if use_environ:
-            for key in os.environ:
-                if key.startswith('GEVENTARES_'):
-                    name = key[11:].lower()
-                    if name:
-                        value = os.environ[key]
-                        kwargs.setdefault(name, value)
+            for setting in config.settings.values():
+                if isinstance(setting, AresSettingMixin):
+                    value = setting.get()
+                    if value is not None:
+                        kwargs.setdefault(setting.kwarg_name, value)
         self.ares = self.ares_class(hub.loop, **kwargs)
         self.pid = os.getpid()
         self.params = kwargs

src/gevent/resolver/dnspython.py

@@ -253,7 +253,9 @@ _is_ipv4_addr = _is_addr
 
 def _is_ipv6_addr(host):
     # Return True if host is a valid IPv6 address
-    host = host.split('%', 1)[0] if host else host
+    if host:
+        s = '%' if isinstance(host, str) else b'%'
+        host = host.split(s, 1)[0]
     return _is_addr(host, dns.ipv6.inet_aton)
 
 class HostsFile(object):

src/gevent/select.py

@@ -11,7 +11,6 @@ from gevent.hub import get_hub
 from gevent.hub import sleep as _g_sleep
 from gevent._compat import integer_types
 from gevent._compat import iteritems
-from gevent._compat import itervalues
 from gevent._util import copy_globals
 from gevent._util import _NONE
 
@@ -200,7 +199,11 @@ if original_poll is not None:
         .. versionadded:: 1.1b1
         """
         def __init__(self):
-            self.fds = {} # {int -> watcher}
+            # {int -> flags}
+            # We can't keep watcher objects in here because people commonly
+            # just drop the poll object when they're done, without calling
+            # unregister(). dnspython does this.
+            self.fds = {}
             self.loop = get_hub().loop
 
         def register(self, fd, eventmask=_NONE):
@@ -216,12 +219,7 @@ if original_poll is not None:
                 # that. Should we raise an error?
 
             fileno = get_fileno(fd)
-            if fileno in self.fds:
-                self.fds[fileno].close()
-
-            watcher = self.loop.io(fileno, flags)
-            watcher.priority = self.loop.MAXPRI
-            self.fds[fileno] = watcher
+            self.fds[fileno] = flags
 
         def modify(self, fd, eventmask):
             self.register(fd, eventmask)
@@ -234,16 +232,23 @@ if original_poll is not None:
                File descriptors that are closed are reported with POLLNVAL.
             """
             result = PollResult()
+            watchers = []
+            io = self.loop.io
+            MAXPRI = self.loop.MAXPRI
             try:
-                for fd, watcher in iteritems(self.fds):
+                for fd, flags in iteritems(self.fds):
+                    watcher = io(fd, flags)
+                    watchers.append(watcher)
+                    watcher.priority = MAXPRI
                     watcher.start(result.add_event, fd, pass_events=True)
                 if timeout is not None and timeout > -1:
                     timeout /= 1000.0
                 result.event.wait(timeout=timeout)
                 return list(result.events)
             finally:
-                for awatcher in itervalues(self.fds):
+                for awatcher in watchers:
                     awatcher.stop()
+                    awatcher.close()
 
         def unregister(self, fd):
             """
@@ -254,8 +259,6 @@ if original_poll is not None:
                library. Previously gevent did nothing.
             """
             fileno = get_fileno(fd)
-            io = self.fds[fileno]
-            io.close()
             del self.fds[fileno]
 
 del original_poll

src/greentest/greentest/__init__.py

@@ -58,6 +58,7 @@ from greentest.sysinfo import CONN_ABORTED_ERRORS
 
 from greentest.skipping import skipOnWindows
 from greentest.skipping import skipOnAppVeyor
+from greentest.skipping import skipOnCI
 from greentest.skipping import skipOnPyPy3OnCI
 from greentest.skipping import skipOnPyPy
 from greentest.skipping import skipOnPyPy3
@@ -100,8 +101,6 @@ from greentest.openfiles import get_number_open_files
 from greentest.openfiles import get_open_files
 
 from greentest.testcase import TestCase
-from greentest.timing import AbstractGenericGetTestCase as GenericGetTestCase
-from greentest.timing import AbstractGenericWaitTestCase as GenericWaitTestCase
 
 from greentest.modules import walk_modules
 

src/greentest/greentest/leakcheck.py

@@ -32,19 +32,15 @@ def ignores_leakcheck(func):
     func.ignore_leakcheck = True
     return func
 
-def wrap_refcount(method):
-    if getattr(method, 'ignore_leakcheck', False):
-        return method
-
-    # Some builtin things that we ignore
-    IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType)
-    try:
+# Some builtin things that we ignore
+IGNORED_TYPES = (tuple, dict, types.FrameType, types.TracebackType)
+try:
     callback_kind = gevent.core.callback
-    except AttributeError:
+except AttributeError:
     # Must be using FFI.
     from gevent._ffi.callback import callback as callback_kind
 
-    def type_hist():
+def _type_hist():
     d = collections.defaultdict(int)
     for x in gc.get_objects():
         k = type(x)
@@ -59,7 +55,7 @@ def wrap_refcount(method):
         d[k] += 1
     return d
 
-    def report_diff(a, b):
+def _report_diff(a, b):
     diff_lines = []
     for k, v in sorted(a.items(), key=lambda i: i[0].__name__):
         if b[k] != v:
@@ -70,6 +66,11 @@ def wrap_refcount(method):
     diff = '\n'.join(diff_lines)
     return diff
 
+def wrap_refcount(method):
+    if getattr(method, 'ignore_leakcheck', False):
+        return method
+
+
     @wraps(method)
     def wrapper(self, *args, **kwargs): # pylint:disable=too-many-branches
         gc.collect()
@@ -78,25 +79,33 @@ def wrap_refcount(method):
         deltas = []
         d = None
         gc.disable()
+
+        # The very first time we are called, we have already been
+        # self.setUp() by the test runner, so we don't need to do it again.
+        needs_setUp = False
+
         try:
             while True:
-
                 # Grab current snapshot
-                hist_before = type_hist()
+                hist_before = _type_hist()
                 d = sum(hist_before.values())
 
+                if needs_setUp:
                     self.setUp()
+                    self.skipTearDown = False
                 try:
                     method(self, *args, **kwargs)
                 finally:
                     self.tearDown()
+                    self.skipTearDown = True
+                    needs_setUp = True
 
                 # Grab post snapshot
                 if 'urlparse' in sys.modules:
                     sys.modules['urlparse'].clear_cache()
                 if 'urllib.parse' in sys.modules:
                     sys.modules['urllib.parse'].clear_cache()
-                hist_after = type_hist()
+                hist_after = _type_hist()
                 d = sum(hist_after.values()) - d
                 deltas.append(d)
 
@@ -119,7 +128,7 @@ def wrap_refcount(method):
                 elif len(deltas) >= 4 and sum(deltas[-4:]) == 0:
                     break
                 elif len(deltas) >= 3 and deltas[-1] > 0 and deltas[-1] == deltas[-2] and deltas[-2] == deltas[-3]:
-                    diff = report_diff(hist_before, hist_after)
+                    diff = _report_diff(hist_before, hist_after)
                     raise AssertionError('refcount increased by %r\n%s' % (deltas, diff))
                 # OK, we don't know for sure yet. Let's search for more
                 if sum(deltas[-3:]) <= 0 or sum(deltas[-4:]) <= 0 or deltas[-4:].count(0) >= 2:
@@ -128,9 +137,10 @@ def wrap_refcount(method):
                 else:
                     limit = 7
                 if len(deltas) >= limit:
-                    raise AssertionError('refcount increased by %r\n%s' % (deltas, report_diff(hist_before, hist_after)))
+                    raise AssertionError('refcount increased by %r\n%s'
+                                         % (deltas,
+                                            _report_diff(hist_before, hist_after)))
         finally:
             gc.enable()
-        self.skipTearDown = True
 
     return wrapper

src/greentest/greentest/params.py

@@ -23,9 +23,11 @@ from greentest.sysinfo import PY3
 from greentest.sysinfo import PYPY
 from greentest.sysinfo import WIN
 from greentest.sysinfo import LIBUV
+from greentest.sysinfo import OSX
 
 from greentest.sysinfo import RUNNING_ON_TRAVIS
 from greentest.sysinfo import EXPECT_POOR_TIMER_RESOLUTION
+from greentest.sysinfo import RESOLVER_ARES
 
 
 # Travis is slow and overloaded; Appveyor used to be faster, but
@@ -60,6 +62,11 @@ if RUNNING_ON_TRAVIS:
     # connected to with the same error.
     DEFAULT_BIND_ADDR = '127.0.0.1'
 
+if RESOLVER_ARES and OSX:
+    # Ares likes to raise "malformed domain name" on '', at least
+    # on OS X
+    DEFAULT_BIND_ADDR = '127.0.0.1'
+
 # For in-process sockets
 DEFAULT_SOCKET_TIMEOUT = 0.1 if not EXPECT_POOR_TIMER_RESOLUTION else 2.0
 

src/greentest/greentest/skipping.py

@@ -23,13 +23,12 @@ import unittest
 
 from greentest import sysinfo
 
+def _identity(f):
+    return f
 
 def _do_not_skip(reason):
     assert reason
-
-    def dec(f):
-        return f
-    return dec
+    return _identity
 
 
 if sysinfo.WIN:
@@ -50,6 +49,11 @@ if sysinfo.RUNNING_ON_APPVEYOR:
 else:
     skipOnAppVeyor = _do_not_skip
 
+if sysinfo.RUNNING_ON_CI:
+    skipOnCI = unittest.skip
+else:
+    skipOnCI = _do_not_skip
+
 if sysinfo.PYPY3 and sysinfo.RUNNING_ON_CI:
     # Same as above, for PyPy3.3-5.5-alpha and 3.5-5.7.1-beta and 3.5-5.8
     skipOnPyPy3OnCI = unittest.skip

src/greentest/greentest/sysinfo.py

@@ -30,8 +30,8 @@ LINUX = sys.platform.startswith('linux')
 OSX = sys.platform == 'darwin'
 
 # XXX: Formalize this better
-LIBUV = os.getenv('GEVENT_CORE_CFFI_ONLY') == 'libuv' or (PYPY and WIN) or hasattr(gevent.core, 'libuv')
-CFFI_BACKEND = bool(os.getenv('GEVENT_CORE_CFFI_ONLY')) or PYPY
+LIBUV = 'libuv' in gevent.core.loop.__module__ # pylint:disable=no-member
+CFFI_BACKEND = PYPY or LIBUV or 'cffi' in os.getenv('GEVENT_LOOP', '')
 
 if '--debug-greentest' in sys.argv:
     sys.argv.remove('--debug-greentest')

src/greentest/greentest/testcase.py

@@ -95,7 +95,9 @@ class TestCaseMetaClass(type):
             if sysinfo.RUN_LEAKCHECKS and timeout is not None:
                 timeout *= 6
         check_totalrefcount = _get_class_attr(classDict, bases, 'check_totalrefcount', True)
+
         error_fatal = _get_class_attr(classDict, bases, 'error_fatal', True)
+        uses_handle_error = _get_class_attr(classDict, bases, 'uses_handle_error', True)
         # Python 3: must copy, we mutate the classDict. Interestingly enough,
         # it doesn't actually error out, but under 3.6 we wind up wrapping
         # and re-wrapping the same items over and over and over.
@@ -108,6 +110,7 @@ class TestCaseMetaClass(type):
                 error_fatal = getattr(value, 'error_fatal', error_fatal)
                 if error_fatal:
                     value = errorhandler.wrap_error_fatal(value)
+                if uses_handle_error:
                     value = errorhandler.wrap_restore_handle_error(value)
                 if check_totalrefcount and sysinfo.RUN_LEAKCHECKS:
                     value = leakcheck.wrap_refcount(value)
@@ -122,6 +125,7 @@ class TestCase(TestCaseMetaClass("NewBase", (TimeAssertMixin, BaseTestCase,), {}
 
     switch_expected = 'default'
     error_fatal = True
+    uses_handle_error = True
     close_on_teardown = ()
 
     def run(self, *args, **kwargs):

src/greentest/greentest/timing.py

@@ -58,6 +58,7 @@ class _DelayWaitMixin(object):
         # otherwise it's the raw number
         seconds = getattr(timeout, 'seconds', timeout)
 
+        gevent.get_hub().loop.update_now()
         start = time.time()
         try:
             result = self.wait(timeout)

src/greentest/test___config.py

+# Copyright 2018 gevent contributors. See LICENSE for details.
+
+import os
+import unittest
+import sys
+
+from gevent import _config
+
+class TestResolver(unittest.TestCase):
+
+    old_resolver = None
+
+    def setUp(self):
+        if 'GEVENT_RESOLVER' in os.environ:
+            self.old_resolver = os.environ['GEVENT_RESOLVER']
+            del os.environ['GEVENT_RESOLVER']
+
+    def tearDown(self):
+        if self.old_resolver:
+            os.environ['GEVENT_RESOLVER'] = self.old_resolver
+
+    def test_key(self):
+        self.assertEqual(_config.Resolver.environment_key, 'GEVENT_RESOLVER')
+
+    def test_default(self):
+        from gevent.resolver.thread import Resolver
+
+        conf = _config.Resolver()
+        self.assertEqual(conf.get(), Resolver)
+
+    def test_env(self):
+        from gevent.resolver.blocking import Resolver
+
+        os.environ['GEVENT_RESOLVER'] = 'foo,bar,block,dnspython'
+
+        conf = _config.Resolver()
+        self.assertEqual(conf.get(), Resolver)
+
+        os.environ['GEVENT_RESOLVER'] = 'dnspython'
+
+        # The existing value is unchanged
+        self.assertEqual(conf.get(), Resolver)
+
+        # A new object reflects it
+        conf = _config.Resolver()
+        from gevent.resolver.dnspython import Resolver as DResolver
+        self.assertEqual(conf.get(), DResolver)
+
+    def test_set_str_long(self):
+        from gevent.resolver.blocking import Resolver
+        conf = _config.Resolver()
+        conf.set('gevent.resolver.blocking.Resolver')
+
+        self.assertEqual(conf.get(), Resolver)
+
+    def test_set_str_short(self):
+        from gevent.resolver.blocking import Resolver
+        conf = _config.Resolver()
+        conf.set('block')
+
+        self.assertEqual(conf.get(), Resolver)
+
+    def test_set_class(self):
+        from gevent.resolver.blocking import Resolver
+        conf = _config.Resolver()
+        conf.set(Resolver)
+
+        self.assertEqual(conf.get(), Resolver)
+
+
+    def test_set_through_config(self):
+        from gevent.resolver.thread import Resolver as Default
+        from gevent.resolver.blocking import Resolver
+
+        conf = _config.Config()
+        self.assertEqual(conf.resolver, Default)
+
+        conf.resolver = 'block'
+        self.assertEqual(conf.resolver, Resolver)
+
+class TestFunctions(unittest.TestCase):
+
+    def test_validate_bool(self):
+        self.assertTrue(_config.validate_bool('on'))
+        self.assertTrue(_config.validate_bool('1'))
+        self.assertFalse(_config.validate_bool('off'))
+        self.assertFalse(_config.validate_bool('0'))
+        self.assertFalse(_config.validate_bool(''))
+
+        with self.assertRaises(ValueError):
+            _config.validate_bool(' hmm ')
+
+    def test_validate_invalid(self):
+        with self.assertRaises(ValueError):
+            _config.validate_invalid(self)
+
+class TestConfig(unittest.TestCase):
+
+    def test__dir__(self):
+        self.assertEqual(sorted(_config.config.settings),
+                         sorted(dir(_config.config)))
+
+    def test_getattr(self):
+        # Bypass the property that might be set here
+        self.assertIsNotNone(_config.config.__getattr__('resolver'))
+
+    def test__getattr__invalid(self):
+        with self.assertRaises(AttributeError):
+            getattr(_config.config, 'no_such_setting')
+
+    def test_set_invalid(self):
+        with self.assertRaises(AttributeError):
+            _config.config.set('no such setting', True)
+
+class TestImportableSetting(unittest.TestCase):
+
+    assertRaisesRegex = getattr(unittest.TestCase, 'assertRaisesRegex',
+                                unittest.TestCase.assertRaisesRegexp)
+    def test_empty_list(self):
+        i = _config.ImportableSetting()
+        with self.assertRaisesRegex(ImportError,
+                                    "Cannot import from empty list"):
+            i._import([])
+
+    def test_path(self):
+        import warnings
+        i = _config.ImportableSetting()
+        path = list(sys.path)
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            with self.assertRaisesRegex(ImportError,
+                                        "Cannot import 'no_such_module'"):
+                i._import('foo/bar/gevent.no_such_module')
+
+        # We restored the path
+        self.assertEqual(path, sys.path)
+
+        self.assertEqual(len(w), 1)
+        self.assertEqual(w[0].category, DeprecationWarning)
+        self.assertIn('Absolute paths', str(w[0].message))
+
+    def test_non_string(self):
+        i = _config.ImportableSetting()
+        self.assertIs(i._import(self), self)
+
+if __name__ == '__main__':
+    unittest.main()

src/greentest/test___example_servers.py

@@ -9,8 +9,8 @@ from unittest import SkipTest
 import socket
 import ssl
 
+import greentest
 from greentest import DEFAULT_XPC_SOCKET_TIMEOUT
-from greentest import main
 from greentest import util
 from greentest import params
 
@@ -101,6 +101,7 @@ class Test_wsgiserver_ssl(Test_wsgiserver):
         ssl_ctx = ssl._create_unverified_context()
 
 
+@greentest.skipOnCI("Timing issues sometimes lead to a connection refused")
 class Test_webproxy(Test_wsgiserver):
     server = 'webproxy.py'
 
@@ -136,4 +137,4 @@ class Test_webproxy(Test_wsgiserver):
 
 
 if __name__ == '__main__':
-    main()
+    greentest.main()

src/greentest/test__api.py

@@ -105,9 +105,10 @@ class TestTimers(greentest.TestCase):
             gevent.sleep(0.02)
 
         gevent.spawn(func)
-        assert lst == [1], lst
-        gevent.sleep(0.03)
-        assert lst == [], lst
+        self.assertEqual(lst, [1])
+        gevent.sleep(0.1)
+        self.assertEqual(lst, [])
+
 
     def test_spawn_is_not_cancelled(self):
         lst = [1]
@@ -116,8 +117,8 @@ class TestTimers(greentest.TestCase):
             gevent.spawn(lst.pop)
             # exiting immediately, but self.lst.pop must be called
         gevent.spawn(func)
-        gevent.sleep(0.01)
-        assert lst == [], lst
+        gevent.sleep(0.1)
+        self.assertEqual(lst, [])
 
 
 if __name__ == '__main__':

src/greentest/test__core_timer.py

 from __future__ import print_function
-from gevent import core
+from gevent import config
 
 from greentest import TestCase
 from greentest import main
@@ -9,15 +9,23 @@ from greentest.sysinfo import CFFI_BACKEND
 
 class Test(TestCase):
     __timeout__ = LARGE_TIMEOUT
+
     repeat = 0
 
     def setUp(self):
         self.called = []
-        self.loop = core.loop(default=True)
+        self.loop = config.loop(default=False)
         self.timer = self.loop.timer(0.001, repeat=self.repeat)
 
-    def tearDown(self):
+    def cleanup(self):
+        # cleanup instead of tearDown to cooperate well with
+        # leakcheck.py
         self.timer.close()
+        # cycle the loop so libuv close callbacks fire
+        self.loop.run()
+        self.loop.destroy()
+        self.loop = None
+        self.timer = None
 
     def f(self, x=None):
         self.called.append(1)
@@ -67,6 +75,8 @@ class TestAgain(Test):
 
         self.assertEqual(x.args, (x,))
 
+        # XXX: On libev, this takes 1 second. On libuv,
+        # it takes the expected time.
         self.loop.run()
 
         self.assertEqual(self.called, [1])

src/greentest/test__event.py

@@ -6,11 +6,13 @@ from gevent.event import Event, AsyncResult
 import greentest
 from greentest.skipping import skipUnderCoverage
 from greentest.six import xrange
+from greentest.timing import AbstractGenericGetTestCase
+from greentest.timing import AbstractGenericWaitTestCase
 
 DELAY = 0.01
 
 
-class TestEventWait(greentest.GenericWaitTestCase):
+class TestEventWait(AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         Event().wait(timeout=timeout)
@@ -19,7 +21,7 @@ class TestEventWait(greentest.GenericWaitTestCase):
         str(Event())
 
 
-class TestWaitEvent(greentest.GenericWaitTestCase):
+class TestWaitEvent(AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         gevent.wait([Event()], timeout=timeout)
@@ -58,19 +60,19 @@ class TestWaitEvent(greentest.GenericWaitTestCase):
         gevent.spawn(waiter).join()
 
 
-class TestAsyncResultWait(greentest.GenericWaitTestCase):
+class TestAsyncResultWait(AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         AsyncResult().wait(timeout=timeout)
 
 
-class TestWaitAsyncResult(greentest.GenericWaitTestCase):
+class TestWaitAsyncResult(AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         gevent.wait([AsyncResult()], timeout=timeout)
 
 
-class TestAsyncResultGet(greentest.GenericGetTestCase):
+class TestAsyncResultGet(AbstractGenericGetTestCase):
 
     def wait(self, timeout):
         AsyncResult().get(timeout=timeout)
@@ -232,5 +234,8 @@ class TestWait_count2(TestWait):
     count = 2
 
 
+del AbstractGenericGetTestCase
+del AbstractGenericWaitTestCase
+
 if __name__ == '__main__':
     greentest.main()

src/greentest/test__greenlet.py

@@ -19,7 +19,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-import sys
+
 import greentest
 import gevent
 import re
@@ -28,6 +28,9 @@ from gevent import greenlet
 from gevent.event import AsyncResult
 from gevent.queue import Queue, Channel
 
+from greentest.timing import AbstractGenericWaitTestCase
+from greentest.timing import AbstractGenericGetTestCase
+
 DELAY = 0.01
 greentest.TestCase.error_fatal = False
 
@@ -38,25 +41,13 @@ class ExpectedError(greentest.ExpectedException):
 
 class TestLink(greentest.TestCase):
 
-    def assertRaises(self, err, func, *args, **kwargs):
-        try:
-            result = func(*args, **kwargs)
-        except:
-            ex = sys.exc_info()[1]
-            if ex is err:
-                return
-            if isinstance(ex, err):
-                return
-            raise
-        raise AssertionError('%s not raised, returned %r' % (err, result))
-
     def test_link_to_asyncresult(self):
         p = gevent.spawn(lambda: 100)
         event = AsyncResult()
         p.link(event)
         self.assertEqual(event.get(), 100)
 
-        for i in range(3):
+        for _ in range(3):
             event2 = AsyncResult()
             p.link(event2)
             self.assertEqual(event2.get(), 100)
@@ -66,12 +57,17 @@ class TestLink(greentest.TestCase):
         p = gevent.spawn(lambda: getcurrent().throw(err))
         event = AsyncResult()
         p.link(event)
-        self.assertRaises(err, event.get)
+        with self.assertRaises(ExpectedError) as exc:
+            event.get()
+
+        self.assertIs(exc.exception, err)
 
-        for i in range(3):
+        for _ in range(3):
             event2 = AsyncResult()
             p.link(event2)
-            self.assertRaises(err, event2.get)
+            with self.assertRaises(ExpectedError) as exc:
+                event2.get()
+            self.assertIs(exc.exception, err)
 
     def test_link_to_queue(self):
         p = gevent.spawn(lambda: 100)
@@ -79,7 +75,7 @@ class TestLink(greentest.TestCase):
         p.link(q.put)
         self.assertEqual(q.get().get(), 100)
 
-        for i in range(3):
+        for _ in range(3):
             p.link(q.put)
             self.assertEqual(q.get().get(), 100)
 
@@ -125,6 +121,8 @@ class TestUnlink(greentest.TestCase):
 
 class LinksTestCase(greentest.TestCase):
 
+    link_method = None
+
     def link(self, p, listener=None):
         getattr(p, self.link_method)(listener)
 
@@ -169,30 +167,33 @@ def sleep0():
 class TestReturn_link(LinksTestCase):
     link_method = 'link'
 
+    p = None
+
     def cleanup(self):
         while self.p._links:
             self.p._links.pop()
+        self.p = None
 
     def test_return(self):
         self.p = gevent.spawn(return25)
         for _ in range(3):
-            self._test_return(self.p, 25, sleep0)
+            self._test_return(self.p, 25)
         self.p.kill()
 
-    def _test_return(self, p, result, action):
+    def _test_return(self, p, result):
         event, queue, callback_flag = self.set_links(p)
 
         # stuff that will time out because there's no unhandled exception:
         xxxxx = self.set_links_timeout(p.link_exception)
 
         sleep(DELAY * 2)
-        assert not p, p
+        self.assertFalse(p)
 
         self.assertEqual(event.get(), result)
         self.assertEqual(queue.get().get(), result)
 
         sleep(DELAY)
-        assert not callback_flag, callback_flag
+        self.assertFalse(callback_flag)
 
         self.check_timed_out(*xxxxx)
 
@@ -202,13 +203,14 @@ class TestReturn_link(LinksTestCase):
 
         p.kill()
         sleep(DELAY)
-        assert not p, p
+        self.assertFalse(p)
+
 
-        assert isinstance(event.get(), greenlet.GreenletExit), event.get()
-        assert isinstance(queue.get().get(), greenlet.GreenletExit), queue.get().get()
+        self.assertIsInstance(event.get(), greenlet.GreenletExit)
+        self.assertIsInstance(queue.get().get(), greenlet.GreenletExit)
 
         sleep(DELAY)
-        assert not callback_flag, callback_flag
+        self.assertFalse(callback_flag)
 
         self.check_timed_out(*xxxxx)
 
@@ -240,7 +242,7 @@ class TestRaise_link(LinksTestCase):
         self.check_timed_out(*xxxxx)
 
     def test_raise(self):
-        p = self.p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_raise')))
+        p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_raise')))
         for _ in range(3):
             self._test_raise(p)
 
@@ -320,22 +322,22 @@ class TestStuff(greentest.TestCase):
         p = gevent.spawn(lambda: 5)
         results = []
 
-        def listener1(*args):
+        def listener1(*_args):
             results.append(10)
             raise ExpectedError('listener1')
 
-        def listener2(*args):
+        def listener2(*_args):
             results.append(20)
             raise ExpectedError('listener2')
 
-        def listener3(*args):
+        def listener3(*_args):
             raise ExpectedError('listener3')
 
         p.link(listener1)
         p.link(listener2)
         p.link(listener3)
         sleep(DELAY * 10)
-        assert results in [[10, 20], [20, 10]], results
+        self.assertIn(results, [[10, 20], [20, 10]])
 
         p = gevent.spawn(lambda: getcurrent().throw(ExpectedError('test_multiple_listeners_error')))
         results = []
@@ -343,7 +345,7 @@ class TestStuff(greentest.TestCase):
         p.link(listener2)
         p.link(listener3)
         sleep(DELAY * 10)
-        assert results in [[10, 20], [20, 10]], results
+        self.assertIn(results, [[10, 20], [20, 10]])
 
     class Results(object):
 
@@ -360,10 +362,10 @@ class TestStuff(greentest.TestCase):
             self.results.append(5)
             raise ExpectedError('listener2')
 
-        def listener3(self, p):
+        def listener3(self, _p):
             raise ExpectedError('listener3')
 
-    def _test_multiple_listeners_error_unlink(self, p, link):
+    def _test_multiple_listeners_error_unlink(self, _p, link):
         # notification must not happen after unlink even
         # though notification process has been already started
         results = self.Results()
@@ -388,20 +390,8 @@ class TestStuff(greentest.TestCase):
         gevent.spawn(e.set, 6)
         self._test_multiple_listeners_error_unlink(e, e.rawlink)
 
-    def test_killing_unlinked(self):
-        e = AsyncResult()
-
-        def func():
-            try:
-                raise ExpectedError('test_killing_unlinked')
-            except:
-                e.set_exception(sys.exc_info()[1])
-                gevent.sleep(0)
 
-        sleep(DELAY)
-
-
-def dummy_test_func(*args):
+def dummy_test_func(*_args):
     pass
 
 
@@ -436,7 +426,7 @@ class TestStr(greentest.TestCase):
         self.assertEqual(str_g, '<Greenlet at X: <bound method A.method of <module.A object at X>>>')
 
 
-class TestJoin(greentest.GenericWaitTestCase):
+class TestJoin(AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         g = gevent.spawn(gevent.sleep, 10)
@@ -446,7 +436,7 @@ class TestJoin(greentest.GenericWaitTestCase):
             g.kill()
 
 
-class TestGet(greentest.GenericGetTestCase):
+class TestGet(AbstractGenericGetTestCase):
 
     def wait(self, timeout):
         g = gevent.spawn(gevent.sleep, 10)
@@ -456,7 +446,7 @@ class TestGet(greentest.GenericGetTestCase):
             g.kill()
 
 
-class TestJoinAll0(greentest.GenericWaitTestCase):
+class TestJoinAll0(AbstractGenericWaitTestCase):
 
     g = gevent.Greenlet()
 
@@ -464,7 +454,7 @@ class TestJoinAll0(greentest.GenericWaitTestCase):
         gevent.joinall([self.g], timeout=timeout)
 
 
-class TestJoinAll(greentest.GenericWaitTestCase):
+class TestJoinAll(AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         g = gevent.spawn(gevent.sleep, 10)
@@ -550,7 +540,7 @@ class TestBasic(greentest.TestCase):
         def func(delay, return_value=4):
             gevent.sleep(delay)
             error = ExpectedError('test_error_exit')
-            error.myattr = return_value
+            setattr(error, 'myattr', return_value)
             raise error
 
         g = gevent.Greenlet(func, 0.001, return_value=5)
@@ -558,13 +548,14 @@ class TestBasic(greentest.TestCase):
         g.rawlink(link_test.append)
         g.start()
         gevent.sleep(0.1)
-        assert not g
-        assert g.dead
-        assert not g.started
-        assert g.ready()
-        assert not g.successful()
-        assert g.value is None  # not changed
-        assert g.exception.myattr == 5
+        self.assertFalse(g)
+        self.assertTrue(g.dead)
+        self.assertFalse(g.started)
+        self.assertTrue(g.ready())
+        self.assertFalse(g.successful())
+        self.assertIsNone(g.value) # not changed
+        self.assertEqual(g.exception.myattr, 5)
+
         assert link_test == [g] or greentest.RUNNING_ON_APPVEYOR, link_test
 
     def _assertKilled(self, g):
@@ -594,7 +585,7 @@ class TestBasic(greentest.TestCase):
         link_test = []
         result = []
         g = gevent.Greenlet(lambda: result.append(1))
-        g.link(lambda x: link_test.append(x))
+        g.link(link_test.append)
         self._test_kill(g, block=block)
         assert not result
         assert link_test == [g]
@@ -609,7 +600,7 @@ class TestBasic(greentest.TestCase):
         result = []
         link_test = []
         g = gevent.Greenlet(lambda: result.append(1))
-        g.link(lambda x: link_test.append(x))
+        g.link(link_test.append)
         g.start()
         self._test_kill(g, block=block)
         assert not result, result
@@ -625,7 +616,7 @@ class TestBasic(greentest.TestCase):
         result = []
         link_test = []
         g = gevent.Greenlet(lambda: result.append(1))
-        g.link(lambda x: link_test.append(x))
+        g.link(link_test.append)
         g.start_later(1)
         self._test_kill(g, block=block)
         assert not result
@@ -639,7 +630,7 @@ class TestBasic(greentest.TestCase):
     def _test_kill_running(self, block):
         link_test = []
         g = gevent.spawn(gevent.sleep, 10)
-        g.link(lambda x: link_test.append(x))
+        g.link(link_test.append)
         self._test_kill(g, block=block)
         gevent.sleep(0.01)
         assert link_test == [g]
@@ -707,5 +698,9 @@ class TestRef(greentest.TestCase):
 
 X = object()
 
+del AbstractGenericGetTestCase
+del AbstractGenericWaitTestCase
+
+
 if __name__ == '__main__':
     greentest.main()

src/greentest/test__hub.py

@@ -20,6 +20,7 @@
 # THE SOFTWARE.
 
 import greentest
+import greentest.timing
 import time
 import re
 import gevent
@@ -71,7 +72,7 @@ class TestExceptionInMainloop(greentest.TestCase):
 
 
 
-class TestSleep(greentest.GenericWaitTestCase):
+class TestSleep(greentest.timing.AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         gevent.sleep(timeout)
@@ -80,7 +81,7 @@ class TestSleep(greentest.GenericWaitTestCase):
         gevent.sleep(0)
 
 
-class TestWaiterGet(greentest.GenericWaitTestCase):
+class TestWaiterGet(greentest.timing.AbstractGenericWaitTestCase):
 
     def setUp(self):
         super(TestWaiterGet, self).setUp()

src/greentest/test__pool.py

 from time import time
 import gevent
-from gevent import pool
+import gevent.pool
 from gevent.event import Event
 from gevent.queue import Queue
-from gevent.timeout import Timeout
+
 import greentest
+import greentest.timing
 import random
 from greentest import ExpectedException
 from greentest import six
@@ -13,12 +14,12 @@ import unittest
 
 
 class TestCoroutinePool(unittest.TestCase):
-    klass = pool.Pool
+    klass = gevent.pool.Pool
 
     def test_apply_async(self):
         done = Event()
 
-        def some_work(x):
+        def some_work(_):
             done.set()
 
         pool = self.klass(2)
@@ -125,18 +126,18 @@ class TestCoroutinePool(unittest.TestCase):
             pool.join()
 
 
-def crash(*args, **kw):
+def crash(*_args, **_kw):
     raise RuntimeError("Whoa")
 
 
 class FakeFile(object):
 
-    def write(*args):
+    def write(self, *_args):
         raise RuntimeError('Whaaa')
 
 
 class PoolBasicTests(greentest.TestCase):
-    klass = pool.Pool
+    klass = gevent.pool.Pool
 
     def test_execute_async(self):
         p = self.klass(size=2)
@@ -208,7 +209,7 @@ class PoolBasicTests(greentest.TestCase):
             second = gevent.spawn(gevent.sleep, 1000)
             try:
                 p.add(first)
-                with self.assertRaises(pool.PoolFull):
+                with self.assertRaises(gevent.pool.PoolFull):
                     p.add(second, blocking=False)
             finally:
                 second.kill()
@@ -223,7 +224,7 @@ class PoolBasicTests(greentest.TestCase):
             second = gevent.spawn(gevent.sleep, 1000)
             try:
                 p.add(first)
-                with self.assertRaises(pool.PoolFull):
+                with self.assertRaises(gevent.pool.PoolFull):
                     p.add(second, timeout=0.100)
             finally:
                 second.kill()
@@ -238,7 +239,7 @@ class PoolBasicTests(greentest.TestCase):
             second = gevent.Greenlet(gevent.sleep, 1000)
             try:
                 p.add(first)
-                with self.assertRaises(pool.PoolFull):
+                with self.assertRaises(gevent.pool.PoolFull):
                     p.start(second, timeout=0.100)
             finally:
                 second.kill()
@@ -304,13 +305,13 @@ if greentest.PYPY and greentest.WIN:
 elif greentest.RUNNING_ON_CI or greentest.EXPECT_POOR_TIMER_RESOLUTION:
     LARGE_RANGE = 100
 
-class TestPool(greentest.TestCase):
+class TestPool(greentest.TestCase): # pylint:disable=too-many-public-methods
     __timeout__ = greentest.LARGE_TIMEOUT
     size = 1
 
     def setUp(self):
         greentest.TestCase.setUp(self)
-        self.pool = pool.Pool(self.size)
+        self.pool = gevent.pool.Pool(self.size)
 
     def cleanup(self):
         self.pool.join()
@@ -438,17 +439,11 @@ class TestPool(greentest.TestCase):
 
         running = [0]
 
-        def short_running_func(i, j):
+        def short_running_func(i, _j):
             running[0] += 1
             return i
 
-        # Send two iterables to make sure varargs and kwargs are handled
-        # correctly
-        for meth in self.pool.imap_unordered, self.pool.imap:
-            running[0] = 0
-            mapping = meth(short_running_func, iterable, iterable,
-                           maxsize=1)
-
+        def make_reader(mapping):
             # Simulate a long running reader. No matter how many workers
             # we have, we will never have a queue more than size 1
             def reader():
@@ -459,7 +454,16 @@ class TestPool(greentest.TestCase):
                     gevent.sleep(0.01)
                     self.assertTrue(len(mapping.queue) <= 2, len(mapping.queue))
                 return result
+            return reader
+
+        # Send two iterables to make sure varargs and kwargs are handled
+        # correctly
+        for meth in self.pool.imap_unordered, self.pool.imap:
+            running[0] = 0
+            mapping = meth(short_running_func, iterable, iterable,
+                           maxsize=1)
 
+            reader = make_reader(mapping)
             l = reader()
             self.assertEqual(sorted(l), iterable)
 
@@ -484,16 +488,16 @@ class TestPool0(greentest.TestCase):
     size = 0
 
     def test_wait_full(self):
-        p = pool.Pool(size=0)
+        p = gevent.pool.Pool(size=0)
         self.assertEqual(0, p.free_count())
         self.assertTrue(p.full())
         self.assertEqual(0, p.wait_available(timeout=0.01))
 
 
-class TestJoinSleep(greentest.GenericWaitTestCase):
+class TestJoinSleep(greentest.timing.AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
-        p = pool.Pool()
+        p = gevent.pool.Pool()
         g = p.spawn(gevent.sleep, 10)
         try:
             p.join(timeout=timeout)
@@ -501,10 +505,10 @@ class TestJoinSleep(greentest.GenericWaitTestCase):
             g.kill()
 
 
-class TestJoinSleep_raise_error(greentest.GenericWaitTestCase):
+class TestJoinSleep_raise_error(greentest.timing.AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
-        p = pool.Pool()
+        p = gevent.pool.Pool()
         g = p.spawn(gevent.sleep, 10)
         try:
             p.join(timeout=timeout, raise_error=True)
@@ -516,7 +520,7 @@ class TestJoinEmpty(greentest.TestCase):
     switch_expected = False
 
     def test(self):
-        p = pool.Pool()
+        p = gevent.pool.Pool()
         res = p.join()
         self.assertTrue(res, "empty should return true")
 
@@ -525,7 +529,7 @@ class TestSpawn(greentest.TestCase):
     switch_expected = True
 
     def test(self):
-        p = pool.Pool(1)
+        p = gevent.pool.Pool(1)
         self.assertEqual(len(p), 0)
         p.spawn(gevent.sleep, 0.1)
         self.assertEqual(len(p), 1)
@@ -535,7 +539,7 @@ class TestSpawn(greentest.TestCase):
         self.assertEqual(len(p), 0)
 
     def testSpawnAndWait(self):
-        p = pool.Pool(1)
+        p = gevent.pool.Pool(1)
         self.assertEqual(len(p), 0)
         p.spawn(gevent.sleep, 0.1)
         self.assertEqual(len(p), 1)
@@ -555,12 +559,12 @@ class TestErrorInIterator(greentest.TestCase):
     error_fatal = False
 
     def test(self):
-        p = pool.Pool(3)
+        p = gevent.pool.Pool(3)
         self.assertRaises(ExpectedException, p.map, lambda x: None, error_iter())
         gevent.sleep(0.001)
 
     def test_unordered(self):
-        p = pool.Pool(3)
+        p = gevent.pool.Pool(3)
 
         def unordered():
             return list(p.imap_unordered(lambda x: None, error_iter()))
@@ -577,11 +581,11 @@ class TestErrorInHandler(greentest.TestCase):
     error_fatal = False
 
     def test_map(self):
-        p = pool.Pool(3)
+        p = gevent.pool.Pool(3)
         self.assertRaises(ZeroDivisionError, p.map, divide_by, [1, 0, 2])
 
     def test_imap(self):
-        p = pool.Pool(1)
+        p = gevent.pool.Pool(1)
         it = p.imap(divide_by, [1, 0, 2])
         self.assertEqual(next(it), 1.0)
         self.assertRaises(ZeroDivisionError, next, it)
@@ -589,7 +593,7 @@ class TestErrorInHandler(greentest.TestCase):
         self.assertRaises(StopIteration, next, it)
 
     def test_imap_unordered(self):
-        p = pool.Pool(1)
+        p = gevent.pool.Pool(1)
         it = p.imap_unordered(divide_by, [1, 0, 2])
         self.assertEqual(next(it), 1.0)
         self.assertRaises(ZeroDivisionError, next, it)

src/greentest/test__queue.py

 import greentest
-from greentest import TestCase, main, GenericGetTestCase
+from greentest import TestCase, main
 import gevent
 from gevent.hub import get_hub, LoopExit
 from gevent import util
 from gevent import queue
 from gevent.queue import Empty, Full
 from gevent.event import AsyncResult
+from greentest.timing import AbstractGenericGetTestCase
 
+# pylint:disable=too-many-ancestors
 
 class TestQueue(TestCase):
 
@@ -372,7 +374,7 @@ class TestJoinEmpty(TestCase):
         q.join()
 
 
-class TestGetInterrupt(GenericGetTestCase):
+class TestGetInterrupt(AbstractGenericGetTestCase):
 
     Timeout = Empty
 
@@ -397,7 +399,7 @@ class TestGetInterruptChannel(TestGetInterrupt):
     kind = queue.Channel
 
 
-class TestPutInterrupt(GenericGetTestCase):
+class TestPutInterrupt(AbstractGenericGetTestCase):
     kind = queue.Queue
     Timeout = Full
 
@@ -430,7 +432,7 @@ class TestPutInterruptChannel(TestPutInterrupt):
         return self.kind()
 
 
-del GenericGetTestCase
+del AbstractGenericGetTestCase
 
 
 if __name__ == '__main__':

src/greentest/test__select.py

@@ -5,18 +5,19 @@ import errno
 from gevent import select, socket
 import gevent.core
 import greentest
+import greentest.timing
 import unittest
 
 
-class TestSelect(greentest.GenericWaitTestCase):
+class TestSelect(greentest.timing.AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         select.select([], [], [], timeout)
 
 
-if sys.platform != 'win32':
 
-    class TestSelectRead(greentest.GenericWaitTestCase):
+@greentest.skipOnWindows("Cant select on files")
+class TestSelectRead(greentest.timing.AbstractGenericWaitTestCase):
 
     def wait(self, timeout):
         r, w = os.pipe()
@@ -46,9 +47,9 @@ if sys.platform != 'win32':
                 self.fail("exception not raised")
 
 
-    if hasattr(select, 'poll'):
-
-        class TestPollRead(greentest.GenericWaitTestCase):
+@unittest.skipUnless(hasattr(select, 'poll'), "Needs poll")
+@greentest.skipOnWindows("Cant poll on files")
+class TestPollRead(greentest.timing.AbstractGenericWaitTestCase):
     def wait(self, timeout):
         # On darwin, the read pipe is reported as writable
         # immediately, for some reason. So we carefully register

src/greentest/test__socket.py

@@ -9,6 +9,7 @@ import unittest
 import greentest
 from functools import wraps
 from greentest import six
+from greentest import LARGE_TIMEOUT
 
 # we use threading on purpose so that we can test both regular and gevent sockets with the same code
 from threading import Thread as _Thread
@@ -42,7 +43,7 @@ class TestTCP(greentest.TestCase):
     __timeout__ = None
     TIMEOUT_ERROR = socket.timeout
     long_data = ", ".join([str(x) for x in range(20000)])
-    if six.PY3:
+    if not isinstance(long_data, bytes):
         long_data = long_data.encode('ascii')
 
     def setUp(self):
@@ -341,7 +342,7 @@ def get_port():
 
 class TestCreateConnection(greentest.TestCase):
 
-    __timeout__ = 5
+    __timeout__ = LARGE_TIMEOUT
 
     def test_refuses(self):
         with self.assertRaises(socket.error) as cm:

src/greentest/test__socket_dns.py

@@ -25,7 +25,9 @@ if getattr(resolver, 'pool', None) is not None:
 
 from greentest.sysinfo import RESOLVER_NOT_SYSTEM
 from greentest.sysinfo import RESOLVER_DNSPYTHON
+from greentest.sysinfo import RESOLVER_ARES
 from greentest.sysinfo import PY2
+import greentest.timing
 
 
 assert gevent_socket.gaierror is socket.gaierror
@@ -444,6 +446,8 @@ class SanitizedHostsFile(HostsFile):
                 continue
             yield name, addr
 
+@greentest.skipIf(greentest.RUNNING_ON_TRAVIS and RESOLVER_ARES,
+                  "This sometimes randomly fails on Travis with ares, beginning Feb 13, 2018")
 class TestEtcHosts(TestCase):
 
     MAX_HOSTS = os.getenv('GEVENTTEST_MAX_ETC_HOSTS', 10)
@@ -567,7 +571,7 @@ add(TestInternational, u'президент.рф', 'russian',
 add(TestInternational, u'президент.рф'.encode('idna'), 'idna')
 
 
-class TestInterrupted_gethostbyname(greentest.GenericWaitTestCase):
+class TestInterrupted_gethostbyname(greentest.timing.AbstractGenericWaitTestCase):
 
     # There are refs to a Waiter in the C code that don't go
     # away yet; one gc may or may not do it.

src/greentest/tests_that_dont_use_resolver.txt

@@ -123,3 +123,5 @@ test_timeout.py
 # implicitly for localhost, which is covered well enough
 # elsewhere that we don't need to spend the 20s (*2)
 test_asyncore.py
+
+test___config.py

tox.ini

@@ -36,7 +36,7 @@ commands =
 basepython =
     python2.7
 setenv =
-	GEVENT_CORE_CFFI_ONLY=1
+	GEVENT_LOOP=libev-cffi
 commands =
     make basictest
 
@@ -44,7 +44,7 @@ commands =
 basepython =
     python2.7
 setenv =
-	GEVENT_CORE_CFFI_ONLY=libuv
+	GEVENT_LOOP=libuv-cffi
 commands =
     make basictest