Add support for pywsgi logging to the stdlib logging module. Fixes #106.

Also documentation updates for the server modules.

.gitignore

@@ -16,6 +16,7 @@ doc/gevent.*.rst
 !doc/gevent.queue.rst
 !doc/gevent.pool.rst
 !doc/gevent.threadpool.rst
+!doc/gevent.wsgi.rst
 
 # Artifacts of configuring in place
 c-ares/config.log

changelog.rst

@@ -82,6 +82,11 @@ Unreleased
 - Fix configuring c-ares for a 32-bit Python when running on a 64-bit
   platform. Reported in :issue:`381` and fixed in :pr:`616` by Chris
   Lane.
+- (Experimental) Let the ``pywsgi.WSGIServer`` accept a
+  ``logging.Logger`` instance for its ``log`` and (new) ``error_log``
+  parameters. Take care that the system is fully monkey-patched very
+  early in the process's lifetime if attempting this, and note that
+  non-file handlers have not been tested. Fixes :issue:`106`.
 
 1.1a2 (Jul 8, 2015)
 ===================

doc/generate_rst.py

@@ -31,11 +31,16 @@ import warnings
 warnings.simplefilter('ignore', DeprecationWarning)
 
 
+def _read(fname, count):
+    with open(fname) as f:
+        return f.read(count)
+
+
 def generate_rst_for_module(module, do=True):
     rst_filename = 'gevent.%s.rst' % module
     exists = os.path.exists(rst_filename)
     if exists:
-        autogenerated = 'autogenerated' in open(rst_filename).read(200).lower()
+        autogenerated = 'autogenerated' in _read(rst_filename, 200).lower()
         if not autogenerated:
             return
     m = __import__('gevent.%s' % module)
@@ -46,9 +51,8 @@ def generate_rst_for_module(module, do=True):
         for line in lines:
             # skip leading blanks. Support both styles of docstrings.
             if line:
-                title = line
+                title = line.strip()
                 break
-        title = title.strip().split('\n')[0]
         title = title.strip(' .')
     prefix = ':mod:`gevent.%s`' % module
     if title:
@@ -60,11 +64,12 @@ def generate_rst_for_module(module, do=True):
     params.update(locals())
     result = template % params
     if exists:
-        if open(rst_filename).read(len(result) + 1) == result:
+        if _read(rst_filename, len(result) + 1) == result:
             return  # already exists one which is the same
     if do:
         print('Generated %s from %s' % (rst_filename, m.__file__))
-        open(rst_filename, 'w').write(result)
+        with open(rst_filename, 'w') as f:
+            f.write(result)
     else:
         print('Would generate %s from %s' % (rst_filename, m.__file__))
 

doc/gevent.threadpool.rst

 
-:mod:`gevent.threadpool`
-========================
+=====================================================
+ :mod:`gevent.threadpool` - A pool of native threads
+=====================================================
 
 .. currentmodule:: gevent.threadpool
 
 .. autoclass:: ThreadPool
+    :inherited-members:
     :members: imap, imap_unordered, map, map_async, apply_async, kill,
               join, spawn
 

doc/gevent.wsgi.rst

+
+==============================================================================
+ :mod:`gevent.wsgi` -- Backwards compatibility alias for :mod:`gevent.pywsgi`
+==============================================================================
+
+In the past, this module used libevent's http support, but that was dropped
+with the introduction of libev. libevent's http support had several
+limitations, including not supporting stream, not supporting
+pipelining, and not supporting SSL.
+
+This module now simply re-exports the contents of the
+:mod:`gevent.pywsgi` module.
+
+.. deprecated:: 1.1
+   Use :mod:`gevent.pywsgi`

doc/reference.rst

@@ -6,26 +6,17 @@ API reference
    gevent
    networking
    synchronization
-   gevent.pool
-   gevent.threadpool
    servers
-   gevent.local
-   gevent.monkey
    gevent.backdoor
-   gevent.baseserver
-   gevent.event
    gevent.fileobject
-   gevent.lock
+   gevent.hub
+   gevent.local
+   gevent.monkey
    gevent.os
    gevent.pool
-   gevent.pywsgi
    gevent.queue
-   gevent.select
    gevent.server
-   gevent.socket
-   gevent.ssl
    gevent.subprocess
    gevent.thread
+   gevent.threadpool
    gevent.util
-   gevent.wsgi
-   gevent.hub

doc/servers.rst

 .. implementing-servers:
 
-Implementing servers
---------------------
+======================
+ Implementing servers
+======================
 
-There are a few classes to simplify server implementation with gevent. They all share the similar interface::
+.. currentmodule:: gevent.baseserver
+
+There are a few classes to simplify server implementation with gevent.
+They all share a similar interface, inherited from :class:`BaseServer`::
 
   def handle(socket, address):
        print('new connection!')
@@ -12,28 +16,33 @@ There are a few classes to simplify server implementation with gevent. They all
   server.start() # start accepting new connections
 
 At this point, any new connection accepted on ``127.0.0.1:1234`` will result in a new
-:class:`Greenlet` spawned using *handle* function. To stop a server use :meth:`stop` method.
+:class:`gevent.Greenlet` spawned running the *handle* function. To stop a server use :meth:`BaseServer.stop` method.
 
-In case of a :class:`WSGIServer`, handle must be a WSGI application callable.
+In case of a :class:`gevent.pywsgi.WSGIServer`, *handle* must be a WSGI application callable.
 
-It is possible to limit the maximum number of concurrent connections, by passing a :class:`Pool` instance::
+It is possible to limit the maximum number of concurrent connections,
+by passing a :class:`gevent.pool.Pool` instance. In addition, passing
+a pool allows the :meth:`BaseServer.stop` method to kill requests that
+are in progress::
 
   pool = Pool(10000) # do not accept more than 10000 connections
   server = StreamServer(('127.0.0.1', 1234), handle, spawn=pool)
   server.serve_forever()
 
-The :meth:`serve_forever` method calls :meth:`start` and then waits until interrupted or until the server is stopped.
 
-The difference between :class:`wsgi.WSGIServer <gevent.wsgi.WSGIServer>` and :class:`pywsgi.WSGIServer <gevent.pywsgi.WSGIServer>`
-is that the first one is very fast as it uses libevent's http server implementation but it shares the issues that
-libevent-http has. In particular:
+.. tip:: If you don't want to limit concurrency, but you *do* want to
+         be able to kill outstanding requests, use a pool created with
+         a size of ``None``.
+
 
-- `does not support streaming`_: the responses are fully buffered in memory before sending; likewise, the incoming requests are loaded in memory in full;
-- `pipelining does not work`_: the server uses ``"Connection: close"`` by default;
-- does not support SSL.
+The :meth:`BaseServer.serve_forever` method calls
+:meth:`BaseServer.start` and then waits until interrupted or until the
+server is stopped.
 
-The :class:`pywsgi.WSGIServer <gevent.pywsgi.WSGIServer>` does not have these limitations.
-In addition, gunicorn_ is a stand-alone server that supports gevent. Gunicorn has its own HTTP parser but can also use :mod:`gevent.wsgi` module.
+The :mod:`gevent.pywsgi` module contains an implementation of a :pep:`3333`
+:class:`WSGI server <gevent.pywsgi.WSGIServer>`. In addition,
+gunicorn_ is a stand-alone server that supports gevent. Gunicorn has
+its own HTTP parser but can also use :mod:`gevent.wsgi` module.
 
 More examples are available in the `code repository`_:
 
@@ -42,8 +51,6 @@ More examples are available in the `code repository`_:
 - `wsgiserver_ssl.py`_ - demonstrates :class:`pywsgi.WSGIServer <gevent.pywsgi.WSGIServer>`
 
 .. _`code repository`: https://github.com/gevent/gevent/tree/master/examples
-.. _`does not support streaming`: http://code.google.com/p/gevent/issues/detail?id=4
-.. _`pipelining does not work`: http://code.google.com/p/gevent/issues/detail?id=32
 .. _gunicorn: http://gunicorn.org
 .. _`echoserver.py`: https://github.com/gevent/gevent/blob/master/examples/echoserver.py#L34
 .. _`wsgiserver.py`: https://github.com/gevent/gevent/blob/master/examples/wsgiserver.py#L18

gevent/baseserver.py

@@ -28,7 +28,7 @@ class BaseServer(object):
         :meth:`set_handle`.
 
         When the request handler returns, the socket used for the
-        request will be closed. (New in gevent 1.1a1.)
+        request will be closed.
 
     :keyword spawn: If provided, is called to create a new
         greenlet to run the handler. By default,
@@ -37,32 +37,36 @@ class BaseServer(object):
 
         - a :class:`gevent.pool.Pool` instance -- ``handle`` will be executed
           using :meth:`gevent.pool.Pool.spawn` only if the pool is not full.
-          While it is full, all the connection are dropped;
+          While it is full, no new connections are accepted;
         - :func:`gevent.spawn_raw` -- ``handle`` will be executed in a raw
-          greenlet which have a little less overhead then :class:`gevent.Greenlet` instances spawned by default;
+          greenlet which has a little less overhead then :class:`gevent.Greenlet` instances spawned by default;
         - ``None`` -- ``handle`` will be executed right away, in the :class:`Hub` greenlet.
-          ``handle`` cannot use any blocking functions as it means switching to the :class:`Hub`.
+          ``handle`` cannot use any blocking functions as it would mean switching to the :class:`Hub`.
         - an integer -- a shortcut for ``gevent.pool.Pool(integer)``
 
+    .. versionchanged:: 1.1a1
+       When the *handle* function returns from processing a connection,
+       the client socket will be closed. This resolves the non-deterministic
+       closing of the socket, fixing ResourceWarnings under Python 3 and PyPy.
 
     """
-    # the number of seconds to sleep in case there was an error in accept() call
-    # for consecutive errors the delay will double until it reaches max_delay
-    # when accept() finally succeeds the delay will be reset to min_delay again
+    #: the number of seconds to sleep in case there was an error in accept() call
+    #: for consecutive errors the delay will double until it reaches max_delay
+    #: when accept() finally succeeds the delay will be reset to min_delay again
     min_delay = 0.01
     max_delay = 1
 
-    # Sets the maximum number of consecutive accepts that a process may perform on
-    # a single wake up. High values give higher priority to high connection rates,
-    # while lower values give higher priority to already established connections.
-    # Default is 100. Note, that in case of multiple working processes on the same
-    # listening value, it should be set to a lower value. (pywsgi.WSGIServer sets it
-    # to 1 when environ["wsgi.multiprocess"] is true)
+    #: Sets the maximum number of consecutive accepts that a process may perform on
+    #: a single wake up. High values give higher priority to high connection rates,
+    #: while lower values give higher priority to already established connections.
+    #: Default is 100. Note, that in case of multiple working processes on the same
+    #: listening value, it should be set to a lower value. (pywsgi.WSGIServer sets it
+    #: to 1 when environ["wsgi.multiprocess"] is true)
     max_accept = 100
 
     _spawn = Greenlet.spawn
 
-    # the default timeout that we wait for the client connections to close in stop()
+    #: the default timeout that we wait for the client connections to close in stop()
     stop_timeout = 1
 
     fatal_errors = (errno.EBADF, errno.EINVAL, errno.ENOTSOCK)
@@ -305,11 +309,19 @@ class BaseServer(object):
         return not hasattr(self, 'socket')
 
     def stop(self, timeout=None):
-        """Stop accepting the connections and close the listening socket.
+        """
+        Stop accepting the connections and close the listening socket.
+
+        If the server uses a pool to spawn the requests, then
+        :meth:`stop` also waits for all the handlers to exit. If there
+        are still handlers executing after *timeout* has expired
+        (default 1 second, :attr:`stop_timeout`), then the currently
+        running handlers in the pool are killed.
 
-        If the server uses a pool to spawn the requests, then :meth:`stop` also waits
-        for all the handlers to exit. If there are still handlers executing after *timeout*
-        has expired (default 1 second), then the currently running handlers in the pool are killed."""
+        If the server does not use a pool, then this merely stops accepting connections;
+        any spawned greenlets that are handling requests continue running until
+        they naturally complete.
+        """
         self.close()
         if timeout is None:
             timeout = self.stop_timeout

gevent/pywsgi.py

 # Copyright (c) 2005-2009, eventlet contributors
-# Copyright (c) 2009-2011, gevent contributors
+# Copyright (c) 2009-2015, gevent contributors
+"""
+A pure-Python, gevent-friendly WSGI server.
 
+The server is provided in :class:`WSGIServer`, but most of the actual
+WSGI work is handled by :class:`WSGIHandler` --- a new instance is
+created for each request. The server can be customized to use
+different subclasses of :class:`WSGIHandler`.
+
+"""
 import errno
 from io import BytesIO
 import string
@@ -19,7 +27,7 @@ from gevent.server import StreamServer
 from gevent.hub import GreenletExit, PY3, reraise
 
 
-__all__ = ['WSGIHandler', 'WSGIServer']
+__all__ = ['WSGIHandler', 'WSGIServer', 'LoggingLogAdapter']
 
 
 MAX_REQUEST_LINE = 8192
@@ -318,6 +326,12 @@ class WSGIHandler(object):
             self.rfile = rfile
 
     def handle(self):
+        """
+        The main request handling method, called by the server.
+
+        This method runs until all requests on the connection have
+        been handled (that is, it implements pipelining).
+        """
         try:
             while self.socket is not None:
                 self.time_start = time.time()
@@ -361,6 +375,12 @@ class WSGIHandler(object):
         return True
 
     def read_request(self, raw_requestline):
+        """
+        Process the incoming request. Parse various headers.
+
+        :raises ValueError: If the request is invalid. This error will
+           not be logged (because it's a client issue, not a server problem).
+        """
         self.requestline = raw_requestline.rstrip()
         words = self.requestline.split()
         if len(words) == 3:
@@ -424,8 +444,9 @@ class WSGIHandler(object):
             message = '%s: %s' % (self.socket, message)
         except Exception:
             pass
+
         try:
-            sys.stderr.write(message + '\n')
+            self.server.error_log.write(message + '\n')
         except Exception:
             traceback.print_exc()
 
@@ -435,7 +456,8 @@ class WSGIHandler(object):
 
         Under both Python 2 and 3, this should return the native
         ``str`` type; under Python 3, this probably means the bytes read
-        from the network need to be decoded.
+        from the network need to be decoded (using the ISO-8859-1 charset, aka
+        latin-1).
         """
         line = self.rfile.readline(MAX_REQUEST_LINE)
         if PY3:
@@ -593,9 +615,7 @@ class WSGIHandler(object):
         return self.write
 
     def log_request(self):
-        log = self.server.log
-        if log:
-            log.write(self.format_request() + '\n')
+        self.server.log.write(self.format_request() + '\n')
 
     def format_request(self):
         now = datetime.now().replace(microsecond=0)
@@ -747,10 +767,91 @@ class WSGIHandler(object):
         return env
 
 
+class _NoopLog(object):
+
+    def write(self, *args, **kwargs):
+        return
+
+
+class LoggingLogAdapter(object):
+    """
+    An adapter for :class:`logging.Logger` instances
+    to let them be used with :class:`WSGIServer`.
+
+    .. warning:: Unless the entire process is monkey-patched at a very
+        early part of the lifecycle (before logging is configured),
+        loggers are likely to not be gevent-cooperative. For example,
+        the socket and syslog handlers use the socket module in a way
+        that can block, and most handlers acquire threading locks.
+
+    .. warning:: It *may* be possible for the logging functions to be
+       called in the :class:`gevent.Hub` greenlet. Code running in the
+       hub greenlet cannot use any gevent blocking functions without triggering
+       a ``LoopExit``.
+
+    .. versionadded:: 1.1a3
+    """
+
+    # gevent avoids importing and using logging because importing it and
+    # creating loggers creates native locks unless monkey-patched.
+
+    def __init__(self, logger, level=20):
+        """
+        Write information to the *logger* at the given *level* (default to INFO).
+        """
+        self.logger = logger
+        self.level = level
+
+    def write(self, msg):
+        self.logger.log(self.level, msg)
+
+    def flush(self):
+        "No-op; required to be a file-like object"
+        pass
+
+    def writelines(self, lines):
+        for line in lines:
+            self.write(line)
+
+
 class WSGIServer(StreamServer):
-    """A WSGI server based on :class:`StreamServer` that supports HTTPS."""
+    """
+    A WSGI server based on :class:`StreamServer` that supports HTTPS.
+
+
+    :keyword log: If given, an object with a ``write`` method to which
+        request logs will be written. If not given, defaults to
+        :obj:`sys.stderr`. You may pass ``None`` to disable request
+        logging. You may use a wrapper, around e.g., :mod:`logging`,
+        to support objects that don't implement a ``write`` method.
+        (If you pass a :class:`logging.Logger` instance, it will be
+        logged to at the :data:`logging.INFO` level.)
+
+    :keyword error_log: If given, a file-like object with a ``write`` method to
+        which error logs will be written. If not given, defaults to
+        :obj:`sys.stderr`. You may pass ``None`` to disable error
+        logging (not recommended). You may use a wrapper, around e.g.,
+        :mod:`logging`, to support objects that don't implement a
+        ``write`` method. (If you pass a :class:`logging.Logger` instance, it will
+        be logged to at the :data:`logging.ERROR` level.)
+
+    .. seealso:: :class:`LoggingLogAdapter`
+
+    .. versionchanged:: 1.1a3
+        Added the ``error_log`` parameter, and set ``wsgi.errors`` in the WSGI
+        environment to this value.
+    """
 
     handler_class = WSGIHandler
+
+    #: The object to which request logs will be written.
+    #: It will never be None.
+    log = None
+
+    #: The object to which error logs will be written.
+    #: It will never be None.
+    error_log = None
+
     base_env = {'GATEWAY_INTERFACE': 'CGI/1.1',
                 'SERVER_SOFTWARE': 'gevent/%d.%d Python/%d.%d' % (gevent.version_info[:2] + sys.version_info[:2]),
                 'SCRIPT_NAME': '',
@@ -759,17 +860,29 @@ class WSGIServer(StreamServer):
                 'wsgi.multiprocess': False,
                 'wsgi.run_once': False}
 
-    def __init__(self, listener, application=None, backlog=None, spawn='default', log='default', handler_class=None,
+    def __init__(self, listener, application=None, backlog=None, spawn='default',
+                 log='default', error_log='default',
+                 handler_class=None,
                  environ=None, **ssl_args):
         StreamServer.__init__(self, listener, backlog=backlog, spawn=spawn, **ssl_args)
         if application is not None:
             self.application = application
         if handler_class is not None:
             self.handler_class = handler_class
-        if log == 'default':
-            self.log = sys.stderr
-        else:
-            self.log = log
+
+        # Note that we can't initialize these as class variables:
+        # sys.stderr might get monkey patched at runtime.
+        def _make_log(l, level=20):
+            if l == 'default':
+                return sys.stderr
+            if l is None:
+                return _NoopLog()
+            if not hasattr(l, 'write') and hasattr(l, 'log'):
+                return LoggingLogAdapter(l, level)
+            return l
+        self.log = _make_log(log)
+        self.error_log = _make_log(error_log, 40) # logging.ERROR
+
         self.set_environ(environ)
         self.set_max_accept()
 
@@ -785,7 +898,7 @@ class WSGIServer(StreamServer):
         if environ_update is not None:
             self.environ.update(environ_update)
         if self.environ.get('wsgi.errors') is None:
-            self.environ['wsgi.errors'] = sys.stderr
+            self.environ['wsgi.errors'] = self.error_log
 
     def set_max_accept(self):
         if self.environ.get('wsgi.multiprocess'):
@@ -799,6 +912,11 @@ class WSGIServer(StreamServer):
         self.update_environ()
 
     def update_environ(self):
+        """
+        Called before the first request is handled to fill in WSGI environment values.
+
+        This includes getting the correct server name and port.
+        """
         address = self.address
         if isinstance(address, tuple):
             if 'SERVER_NAME' not in self.environ:
@@ -815,5 +933,10 @@ class WSGIServer(StreamServer):
             self.environ.setdefault('SERVER_PORT', '')
 
     def handle(self, socket, address):
+        """
+        Create an instance of :attr:`handler_class` to handle the request.
+
+        This method blocks until the handler returns.
+        """
         handler = self.handler_class(socket, address, self)
         handler.handle()

gevent/wsgi.py

+"""Backwards compatibility alias for :mod:`gevent.pywsgi`.
+
+In the past, this used libevent's http support, but that was dropped
+with the introduction of libev. libevent's http support had several
+limitations, including not supporting stream, not supporting
+pipelining, and not supporting SSL.
+
+.. deprecated:: 1.1
+   Use :mod:`gevent.pywsgi`
+"""
+
 from gevent.pywsgi import *
 import gevent.pywsgi as _pywsgi
 __all__ = _pywsgi.__all__

greentest/test__pywsgi.py

@@ -236,7 +236,10 @@ class TestCase(greentest.TestCase):
     application = None
 
     def init_server(self, application):
-        self.server = pywsgi.WSGIServer(('127.0.0.1', 0), application)
+        import logging
+        logger = logging.getLogger('gevent.pywsgi')
+        self.server = pywsgi.WSGIServer(('127.0.0.1', 0), application,
+                                        log=logger, error_log=logger)
 
     def setUp(self):
         application = self.application
@@ -847,7 +850,7 @@ class TestEmptyYield(TestCase):
         read_http(fd, body='', chunks=chunks)
 
         garbage = fd.read()
-        self.assert_(garbage == b"", "got garbage: %r" % garbage)
+        self.assertEqual(garbage, b"", "got garbage: %r" % garbage)
 
 
 class TestFirstEmptyYield(TestCase):
@@ -886,7 +889,7 @@ class TestEmptyYield304(TestCase):
         fd.write('GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n')
         read_http(fd, code=304, body='', chunks=False)
         garbage = fd.read()
-        self.assert_(garbage == b"", "got garbage: %r" % garbage)
+        self.assertEqual(garbage, b"", "got garbage: %r" % garbage)
 
 
 class TestContentLength304(TestCase):
@@ -907,7 +910,7 @@ class TestContentLength304(TestCase):
         body = "Invalid Content-Length for 304 response: '100' (must be absent or zero)"
         read_http(fd, code=200, reason='Raised', body=body, chunks=False)
         garbage = fd.read()
-        self.assertTrue(garbage == b"", "got garbage: %r" % garbage)
+        self.assertEqual(garbage, b"", "got garbage: %r" % garbage)
 
 
 class TestBody304(TestCase):