Fleshing out testing and building documentation.

Deprecate the old EMBED vars in favor of GEVENTSETUP_EMBED.

Refs #1402

.travis.yml

@@ -19,7 +19,7 @@ env:
         - CCACHE_NOHASHDIR=true
         - BUILD_LIBS=$HOME/.libs
         - CFLAGS="-g -pipe"
-        - CPPFLAGS="-I$BUILD_LIBS/include"
+        - CPPFLAGS="-I$BUILD_LIBS/include -DEV_VERIFY=3"
         - LDFLAGS="-L$BUILD_LIBS/lib"
         - LD_LIBRARY_PATH="$BUILD_LIBS/lib"
 
@@ -42,7 +42,7 @@ env:
       - TRAVIS_PYTHON_VERSION=3.7
       - TRAVIS_PYTHON_VERSION=pypy2.7
       - TRAVIS_PYTHON_VERSION=pypy3.6
-      - TRAVIS_PYTHON_VERSION=2.7 EMBED=0
+      - TRAVIS_PYTHON_VERSION=2.7 GEVENTSETUP_EMBED=0
 
 matrix:
   fast_finish: true
@@ -110,7 +110,7 @@ jobs:
         # so that we don't have to use build isolation and can better use the cache;
         # Note that we can't use -U for cffi and greenlet on PyPy.
         - &build-gevent-deps pip install -U setuptools wheel && pip install cffi cython greenlet
-        - GEVENTSETUP_EV_VERIFY=3 pip install --no-build-isolation .[test,events,dnspython]
+        - pip install --no-build-isolation .[test]
       script: ccache -s
       before_script: true
       after_success: true
@@ -125,13 +125,13 @@ jobs:
     - <<: *build-gevent
       env: TRAVIS_PYTHON_VERSION=pypy3.6
     - <<: *build-gevent
-      env: TRAVIS_PYTHON_VERSION=2.7 EMBED=0
+      env: TRAVIS_PYTHON_VERSION=2.7 GEVENTSETUP_EMBED=0
       install:
         # Install the Python runtime
         - *build-gevent-python
         - *build-gevent-deps
         # Install the C dependencies to a known location. This is used
-        # to test "no embed" cases. It might seem like it would prime
+        # to test 'no embed' cases. It might seem like it would prime
         # the CCache for when we *do* embed if we did it as part of the generic build stage,
         # but overall it just seems to slow things down. The Travis caching and CCache is not working as
         # well as we would hope.
@@ -142,7 +142,7 @@ jobs:
         # delete to avoid repacking the archive
         - rm -rf $BUILD_LIBS/share/man/
         - ls -l $BUILD_LIBS $BUILD_LIBS/lib
-        - EMBED=0 pip install --no-build-isolation .[test,events,dnspython]
+        - pip install --no-build-isolation .[test]
 
     # Ok, now we switch to the test stage. These are all in addition
     # to the jobs created by the matrix (and should override the `script` command).
@@ -236,16 +236,19 @@ jobs:
       name: pure37
 
 
-    # 2.7, no-embed. Run the tests that exercise the libraries we linked to.
+    # 2.7, no-embed. Run the tests that exercise the libraries we
+    # linked to.
+    # XXX: The CFFI backends, as exercised by test-lib[eu]v-jobs
+    # don't really support this!
     - <<: *test-ares-jobs
-      env: TRAVIS_PYTHON_VERSION=2.7 EMBED=0
+      env: TRAVIS_PYTHON_VERSION=2.7 GEVENTSETUP_EMBED=0
       name: ares27-noembed
-    - <<: *test-libuv-jobs
-      env: TRAVIS_PYTHON_VERSION=2.7 EMBED=0
-      name: libuv27-noembed
-    - <<: *test-libev-jobs
-      env: TRAVIS_PYTHON_VERSION=2.7 EMBED=0
-      name: libev27-noembed
+    # - <<: *test-libuv-jobs
+    #   env: TRAVIS_PYTHON_VERSION=2.7 GEVENTSETUP_EMBED=0
+    #   name: libuv27-noembed
+    # - <<: *test-libev-jobs
+    #   env: TRAVIS_PYTHON_VERSION=2.7 GEVENTSETUP_EMBED=0
+    #   name: libev27-noembed
 
     # PyPy 2.7
     - <<: *test-dnspython-jobs

CHANGES.rst

@@ -86,6 +86,10 @@
   -a``, respectively. The remainder of the ``Makefile`` contained
   Travis CI commands that have been moved to ``.travis.yml``.
 
+- Deprecate the ``EMBED`` and ``LIBEV_EMBED``, etc, build-time environment
+  variables. Instead, use ``GEVENTSETUP_EMBED`` and
+  ``GEVENTSETUP_EMBED_LIBEV``. See :issue:`1402`.
+
 1.4.0 (2019-01-04)
 ==================
 

CONTRIBUTING.rst

-Basics
-======
+========================
+ Contributing to gevent
+========================
 
-Please see `contribution-guide.org <http://www.contribution-guide.org/>`_ for
-general details on what we expect from contributors. Thanks!
+Please see `contribution-guide.org
+<http://www.contribution-guide.org/>`_ for general details on what we
+need from contributors.
+
+If you're filing a bug that needs a code example, please be sure it's
+a `Short, Self Contained, Correct, Example <http://sscce.org>`_
+
+Thanks!
 
 
 gevent-specific details
 =======================
 
+For information on building gevent, and adding and updating test
+cases, see `the development documentation
+<http://www.gevent.org/contents.html#contents-developing>`_.
+
 There are a number of systems in place to help ensure gevent is of the
 highest possible quality:
 
-- Builds on Travis CI automatically submit updates to `coveralls.io`_ to
-  monitor test coverage. Pull requests that don't feature adequate test
-  coverage will be automatically failed.
-
-.. image:: https://coveralls.io/repos/gevent/gevent/badge.svg?branch=master&service=github
-   :target: https://coveralls.io/github/gevent/gevent?branch=master
-
-- Likewise, builds on Travis CI will automatically submit updates to
-  `landscape.io`_ to monitor code health (adherence to PEP8, absence of
-  common code smells, etc). Pull requests that decrease code health will
-  be automatically failed.
-
-.. image:: https://landscape.io/github/gevent/gevent/master/landscape.svg?style=flat
-   :target: https://landscape.io/github/gevent/gevent/master
-   :alt: Code Health
-
 - A test suite is run for every push and pull request submitted. Travis
   CI is used to test on Linux, and `AppVeyor`_ runs the builds on
   Windows. Pull requests with tests that don't pass will be
   automatically failed.
 
-.. image:: https://travis-ci.org/gevent/gevent.svg?branch=master
+
+  .. image:: https://travis-ci.org/gevent/gevent.svg?branch=master
      :target: https://travis-ci.org/gevent/gevent
 
-.. image:: https://ci.appveyor.com/api/projects/status/q4kl21ng2yo2ixur?svg=true
+  .. image:: https://ci.appveyor.com/api/projects/status/q4kl21ng2yo2ixur?svg=true
      :target: https://ci.appveyor.com/project/denik/gevent
 
-.. _landscape.io: https://landscape.io/github/gevent/gevent
+- Builds on Travis CI automatically submit updates to `coveralls.io`_ to
+  monitor test coverage. Pull requests that don't feature adequate test
+  coverage will be automatically failed.
+
+  .. image:: https://coveralls.io/repos/gevent/gevent/badge.svg?branch=master&service=github
+     :target: https://coveralls.io/github/gevent/gevent?branch=master
+
+- Travis CI builds also run `pylint
+  <https://pylint.readthedocs.io/en/latest/>`_ to enforce code quality
+  conventions (PEP8 compliance and the like).
+
+
 .. _coveralls.io: https://coveralls.io/github/gevent/gevent
 .. _AppVeyor: https://ci.appveyor.com/project/denik/gevent
 

_setuputils.py

@@ -70,7 +70,12 @@ def glob_many(*globs):
 
 ## Configuration
 
-def _parse_environ(key):
+# Environment variables that are intended to be used outside of our own
+# CI should be documented in ``installing_from_source.rst``.
+# They should all begin with ``GEVENTSETUP_``
+
+
+def _bool_from_environ(key):
     value = os.environ.get(key)
     if not value:
         return
@@ -82,33 +87,50 @@ def _parse_environ(key):
     raise ValueError('Environment variable %r has invalid value %r. '
                      'Please set it to 1, 0 or an empty string' % (key, value))
 
-IGNORE_CFFI = _parse_environ("GEVENT_NO_CFFI_BUILD")
+IGNORE_CFFI = _bool_from_environ("GEVENTSETUP_NO_CFFI_BUILD")
 
-def _get_config_value(key, defkey, path=None):
+def _check_embed(key, defkey, path=None, warn=False):
     """
     Find a boolean value, configured in the environment at *key* or
     *defkey* (typically, *defkey* will be shared by several calls). If
     those don't exist, then check for the existence of *path* and return
     that (if path is given)
     """
-    value = _parse_environ(key)
+    value = _bool_from_environ(key)
     if value is None:
-        value = _parse_environ(defkey)
+        value = _bool_from_environ(defkey)
     if value is not None:
+        if warn:
+            print("Warning: gevent setup: legacy environment key %s or %s found"
+                  % (key, defkey))
         return value
-    return os.path.exists(path) if path is not None else False
+    return os.path.exists(path) if path is not None else None
 
 def should_embed(dep_name):
     """
-    Check the configuration for the dep_name and see if it
-    should be embedded. Environment keys are derived from the
-    dep name: libev becomes LIBEV_EMBED and c-ares becomes CARES_EMBED.
+    Check the configuration for the dep_name and see if it should be
+    embedded. Environment keys are derived from the dep name: libev
+    becomes GEVENTSETUP_EMBED_LIBEV and c-ares becomes
+    GEVENTSETUP_EMBED_CARES.
     """
     path = dep_abspath(dep_name)
-    defkey = 'EMBED'
-    key = dep_name.replace('-', '').upper() + '_' + defkey
+    normal_dep_key = dep_name.replace('-', '').upper()
+
+    default_key = 'GEVENTSETUP_EMBED'
+    dep_key = default_key + '_' + normal_dep_key
+
+    result = _check_embed(dep_key, default_key)
+    if result is not None:
+        return result
+
+    # Not defined, check legacy settings, and fallback to the path
+
+    legacy_default_key = 'EMBED'
+    legacy_dep_key = normal_dep_key + '_' + legacy_default_key
+
 
-    return _get_config_value(key, defkey, path)
+    return _check_embed(legacy_dep_key, legacy_default_key, path,
+                        warn=True)
 
 ## Headers
 

docs/conf.py

@@ -39,10 +39,15 @@ extensions = [
     'sphinx.ext.doctest',
     'sphinx.ext.coverage',
     'sphinx.ext.intersphinx',
-    'mysphinxext',
     'sphinx.ext.extlinks',
     'sphinx.ext.viewcode',
+
+    # Third-party
     'repoze.sphinx.autointerface',
+    'sphinxcontrib.programoutput',
+
+    # Ours
+    'mysphinxext',
 ]
 
 intersphinx_mapping = {

docs/contents.rst

@@ -11,8 +11,6 @@ Introduction and Basics
    install
    intro
    whatsnew_1_5
-   whatsnew_1_4
-   whatsnew_1_3
    api/gevent
    api/gevent.greenlet
    servers
@@ -21,8 +19,6 @@ Introduction and Basics
    loop_impls
    configuration
    changelog
-   installing_from_source
-   development
 
 API Details
 ===========
@@ -32,6 +28,20 @@ API Details
 
    api/index
 
+.. This anchor is referenced from CONTRIBUTING.rst
+
+.. _contents-developing:
+
+Developing and Packaging gevent
+===============================
+
+.. toctree::
+   :maxdepth: 2
+
+   installing_from_source
+   development
+
+
 Related Information
 ===================
 

docs/development.rst

@@ -2,6 +2,16 @@
  Development
 =============
 
+..
+    The contributor guide (CONTRIBUTING.rst) references this document.
+
+    Things to include:
+
+    - Avoiding hard test dependencies.
+    - Resource usage.
+    - Custom commands in ``setup.py``
+
+
 To install the latest development version::
 
   pip install git+git://github.com/gevent/gevent.git#egg=gevent
@@ -42,6 +52,8 @@ run::
 
   (env) $ python -mgevent.tests
 
+.. command-output:: python -mgevent.tests --help
+
 Before submitting a pull request, it's a good idea to run the tests
 across all supported versions of Python, and to check the code quality
 using prospector. This is what is done on Travis CI. Locally it

docs/installing_from_source.rst

@@ -5,9 +5,12 @@
 If you are unable to use the binary wheels (for platforms where no
 pre-built wheels are available or if wheel installation is disabled),
 you can build gevent from source. A normal ``pip install`` will
-fallback to doing this if no binary wheel is available. See
-`Installing From Source <installing-from-source>`_ for more.
+fallback to doing this if no binary wheel is available. (If you'll be
+:ref:`developing <development>` gevent, you'll need to install from
+source also; follow that link for more details.)
 
+General Notes
+=============
 
 - You can force installation of gevent from source with ``pip
   install --no-binary gevent gevent``. This is useful if there is a
@@ -28,9 +31,9 @@ fallback to doing this if no binary wheel is available. See
   `have issues installing gevent for this reason
   <https://github.com/pypa/pipenv/issues/2113>`_.
 
-- gevent comes with a ``pyproject.toml`` file that installs the build
-  dependencies, including CFFI (needed for libuv support). pip 18 or
-  above is required for this support.
+- gevent 1.5 comes with a ``pyproject.toml`` file that installs the
+  build dependencies, including CFFI (needed for libuv support). pip
+  18 or above is required for this support.
 
 
 Common Installation Issues
@@ -48,8 +51,9 @@ those compiling gevent from source.
   <https://github.com/gevent/gevent/issues/570>`_ and `issue #612
   <https://github.com/gevent/gevent/issues/612>`_ for examples.
 
-- Also check for conflicts with environment variables like ``CFLAGS``. For
-  example, see `Library Updates <http://www.gevent.org/whatsnew_1_1.html#library-updates-label>`_.
+- Also check for conflicts with environment variables like ``CFLAGS``.
+  For example, see `Library Updates
+  <http://www.gevent.org/whatsnew_1_1.html#library-updates-label>`_.
 
 - Users of a recent SmartOS release may need to customize the
   ``CPPFLAGS`` (the environment variable containing the default
@@ -63,3 +67,72 @@ those compiling gevent from source.
   == 'CPython'", 'at', " ; sys_platform == 'win32' and
   platform_python_implementation == 'CPython'")``, the version of
   setuptools is too old. Install a more recent version of setuptools.
+
+Build-Time Configuration
+========================
+
+There are a few knobs that can be tweaked at gevent build time. These
+are mostly useful for downstream packagers. They all take the form of
+environment variables that must be set when ``setup.py`` is called
+(note that ``pip install`` will invoke ``setup.py``). Toggle flags
+that have boolean values may take the form of 0/1, true/false, off/on,
+yes/no.
+
+``CPPFLAGS``
+  A standard variable used when building the C extensions. gevent may
+  make slight modifications to this variable.
+``CFLAGS``
+  A standard variable used when building the C extensions. gevent may
+  make slight modifications to this variable.
+``LDFLAGS``
+  A standard variable used when building the C extensions. gevent may
+  make slight modifications to this variable.
+``GEVENTSETUP_EV_VERIFY``
+  If set, the value is passed through as the value of the
+  ``EV_VERIFY`` C compiler macro when libev is embedded.
+
+  In general, setting ``CPPFLAGS`` is more general and can contain
+  other macros recognized by libev.
+
+``GEVENTSETUP_NO_CFFI_BUILD``
+  A boolean; when set to true, this disables all attempts at building
+  the CFFI modules. *This disables libuv.* (TODO: verify that.)
+  Ignored on PyPy and ignored on Windows.
+
+
+Embedding Libraries
+-------------------
+
+By default, gevent builds and embeds tested versions of its
+C dependencies libev, libuv, and c-ares. This is the
+recommended configuration as the specific versions used are tested by
+gevent, and sometimes require patches to be applied. Moreover,
+embedding, especially in the case of libev, can be more efficient as
+features not needed by gevent can be disabled, resulting in smaller or
+faster libraries or runtimes.
+
+However, this can be disabled (TODO: verify how this interacts with
+CFFI; see NO_CFFI_BUILD), either for all libraries at once or for
+individual libraries.
+
+When embedding a library is disabled, the library must already be
+installed on the system in a way the compiler can access and link
+(i.e., correct ``CPPFLAGS``, etc) in order to use the corresponding C
+extension.
+
+``GEVENTSETUP_EMBED``
+  A boolean defaulting to true. When turned off (e.g.,
+  ``GEVENTSETUP_EMBED=0``), libraries are not embedded in the gevent C
+  extensions. The value of this is used as the default for all the
+  libraries if no more specific version is defined.
+``GEVENTSETUP_EMBED_LIBEV``
+  Controls embedding libev.
+``GEVENTSETUP_EMBED_CARES``
+  Controls embedding c-ares.
+``GEVENTSETUP_EMBED_LIBUV``
+  This is not defined or used, only a CFFI extension is available and
+  those are always embedded.
+
+Older versions of gevent supported ``EMBED`` and ``LIBEV_EMBED``, etc,
+to mean the same thing. Those aliases still work but are deprecated
+and print a warning.

docs/intro.rst

@@ -48,6 +48,12 @@ in a multi-greenlet environment.
 
 See :doc:`examples/concurrent_download`.
 
+
+.. tip::
+
+   Insight into the monkey-patching process can be obtained by
+   observing the events :mod:`gevent.monkey` emits.
+
 Beyond sockets
 --------------
 

docs/monitoring.rst

@@ -57,6 +57,11 @@ be emitted.
 Visibility
 ==========
 
+.. tip::
+
+   Insight into the monkey-patching process can be obtained by
+   observing the events :mod:`gevent.monkey` emits.
+
 It is sometimes useful to get an overview of all existing greenlets
 and their stack traces. The function
 :func:`gevent.util.print_run_info` will collect this info and print it

docs/older_releases.rst

@@ -3,8 +3,10 @@
 ==================================
 
 .. toctree::
+   :maxdepth: 2
 
-
+   whatsnew_1_4
+   whatsnew_1_3
    whatsnew_1_2
    whatsnew_1_1
    whatsnew_1_0

setup.py

@@ -23,7 +23,8 @@ from _setuputils import GeventClean
 from _setuputils import BuildFailed
 from _setuputils import cythonize1
 
-
+# Environment variables that are intended to be used outside of our own
+# CI should be documented in ``installing_from_source.rst``
 
 if WIN:
     # Make sure the env vars that make.cmd needs are set
@@ -283,10 +284,13 @@ for mod in _to_cythonize:
 del _to_cythonize
 
 
-if IGNORE_CFFI and not PYPY:
+if IGNORE_CFFI and not PYPY and not WIN:
     # Allow distributors to turn off CFFI builds
     # even if it's available, because CFFI always embeds
     # our copy of libev/libuv and they may not want that.
+    # Not allowed on PyPy and not allowed on Windows, because those
+    # backends are required there.
+    # TODO: CONFIRM if this is still the case.
     del cffi_modules[:]
 
 ## Extras
@@ -394,6 +398,7 @@ def run_setup(ext_modules, run_make):
             # End end-user extras
             'docs': [
                 'repoze.sphinx.autointerface',
+                'sphinxcontrib-programoutput',
             ],
             # To the extent possible, we should work to make sure
             # our tests run, at least a basic set, without any of

src/gevent/monkey.py

@@ -45,8 +45,10 @@ module provides functions for that purpose.
 - :func:`is_object_patched`
 - :func:`get_original`
 
-Plugins
-=======
+.. _plugins:
+
+Plugins and Events
+==================
 
 Beginning in gevent 1.3, events are emitted during the monkey patching process.
 These events are delivered first to :mod:`gevent.events` subscribers, and then