Make all modules that copy globals do so using the same function

This serves to be better documenting about what they're doing and why,
instead of having a dozen slightly different loops that each have to be
examined.

gevent/_socket2.py

@@ -7,12 +7,12 @@ Python 2 socket module.
 
 import time
 from gevent import _socketcommon
+from gevent._util import copy_globals
 from gevent._compat import PYPY
 
-for key in _socketcommon.__dict__:
-    if key.startswith('__') or key in _socketcommon.__py3_imports__ or key in _socketcommon.__extensions__:
-        continue
-    globals()[key] = getattr(_socketcommon, key)
+copy_globals(_socketcommon, globals(),
+             names_to_ignore=_socketcommon.__py3_imports__ + _socketcommon.__extensions__,
+             dunder_names_to_keep=())
 
 __socket__ = _socketcommon.__socket__
 __implements__ = _socketcommon._implements

gevent/_socket3.py

@@ -6,19 +6,19 @@ Python 3 socket module.
 # pylint: disable=undefined-variable
 # pylint: disable=too-many-statements,too-many-branches
 # pylint: disable=too-many-public-methods,unused-argument
-
+from __future__ import absolute_import
 import io
 import os
 import sys
 import time
 from gevent import _socketcommon
+from gevent._util import copy_globals
 import _socket
 from os import dup
 
-for key in _socketcommon.__dict__:
-    if key.startswith('__') or key in _socketcommon.__extensions__:
-        continue
-    globals()[key] = getattr(_socketcommon, key)
+copy_globals(_socketcommon, globals(),
+             names_to_ignore=_socketcommon.__extensions__,
+             dunder_names_to_keep=())
 
 __socket__ = _socketcommon.__socket__
 __implements__ = _socketcommon._implements

gevent/_socketcommon.py

@@ -74,6 +74,8 @@ from gevent.hub import get_hub
 from gevent.hub import ConcurrentObjectUseError
 from gevent.timeout import Timeout
 from gevent._compat import string_types, integer_types
+from gevent._util import copy_globals
+from gevent._util import _NONE
 
 is_windows = sys.platform == 'win32'
 # pylint:disable=no-name-in-module,unused-import
@@ -105,12 +107,9 @@ _realsocket = _socket.socket
 import socket as __socket__
 
 _name = _value = None
-for _name in __imports__[:]:
-    try:
-        _value = getattr(__socket__, _name)
-        globals()[_name] = _value
-    except AttributeError:
-        __imports__.remove(_name)
+__imports__ = copy_globals(__socket__, globals(),
+                           only_names=__imports__,
+                           ignore_missing_names=True)
 
 for _name in __socket__.__all__:
     _value = getattr(__socket__, _name)
@@ -120,13 +119,6 @@ for _name in __socket__.__all__:
 
 del _name, _value
 
-
-class _NONE(object):
-
-    def __repr__(self):
-        return "<default value>"
-
-_NONE = _NONE()
 _timeout_error = timeout # pylint: disable=undefined-variable
 
 

gevent/_ssl2.py

@@ -21,13 +21,8 @@ from gevent._socket2 import socket
 from gevent.socket import _fileobject, timeout_default
 from gevent.socket import error as socket_error, EWOULDBLOCK
 from gevent.socket import timeout as _socket_timeout
-from gevent._compat import string_types, PYPY
-
-try:
-    long
-except NameError:
-    # Make us importable under Py3k for documentation
-    long = int
+from gevent._compat import PYPY
+from gevent._util import copy_globals
 
 
 __implements__ = ['SSLSocket',
@@ -35,30 +30,13 @@ __implements__ = ['SSLSocket',
                   'get_server_certificate',
                   'sslwrap_simple']
 
-__imports__ = ['SSLError',
-               'RAND_status',
-               'RAND_egd',
-               'RAND_add',
-               'cert_time_to_seconds',
-               'get_protocol_name',
-               'DER_cert_to_PEM_cert',
-               'PEM_cert_to_DER_cert']
+# Import all symbols from Python's ssl.py, except those that we are implementing
+# and "private" symbols.
+__imports__ = copy_globals(__ssl__, globals(),
+                           # SSLSocket *must* subclass gevent.socket.socket; see issue 597
+                           names_to_ignore=__implements__ + ['socket', 'namedtuple'],
+                           dunder_names_to_keep=())
 
-for name in __imports__[:]:
-    try:
-        value = getattr(__ssl__, name)
-        globals()[name] = value
-    except AttributeError:
-        __imports__.remove(name)
-
-for name in dir(__ssl__):
-    if not name.startswith('_'):
-        value = getattr(__ssl__, name)
-        if isinstance(value, (int, long, tuple)) or isinstance(value, string_types):
-            globals()[name] = value
-            __imports__.append(name)
-
-del name, value
 
 # Py2.6 can get RAND_status added twice
 __all__ = list(set(__implements__) | set(__imports__))

gevent/_ssl3.py

@@ -18,6 +18,7 @@ import errno
 from gevent.socket import socket, timeout_default
 from gevent.socket import error as socket_error
 from gevent.socket import timeout as _socket_timeout
+from gevent._util import copy_globals
 
 from weakref import ref as _wref
 
@@ -28,22 +29,12 @@ __implements__ = [
     'get_server_certificate',
 ]
 
-__imports__ = []
-
-_name = _value = None
-for _name in dir(__ssl__):
-    if _name in __implements__:
-        continue
-    if _name.startswith('__'):
-        continue
-    if _name == 'socket':
+# Import all symbols from Python's ssl.py, except those that we are implementing
+# and "private" symbols.
+__imports__ = copy_globals(__ssl__, globals(),
                            # SSLSocket *must* subclass gevent.socket.socket; see issue 597
-        continue
-    _value = getattr(__ssl__, _name)
-    globals()[_name] = _value
-    __imports__.append(_name)
-
-del _name, _value
+                           names_to_ignore=__implements__ + ['socket', 'namedtuple'],
+                           dunder_names_to_keep=())
 
 __all__ = __implements__ + __imports__
 

gevent/_sslgte279.py

@@ -23,6 +23,7 @@ from gevent.socket import timeout_default
 from gevent.socket import error as socket_error
 from gevent.socket import timeout as _socket_timeout
 from gevent._compat import PYPY
+from gevent._util import copy_globals
 
 __implements__ = [
     'SSLContext',
@@ -35,24 +36,12 @@ __implements__ = [
     '_create_stdlib_context',
 ]
 
-__imports__ = []
-
 # Import all symbols from Python's ssl.py, except those that we are implementing
 # and "private" symbols.
-_name = _value = None
-for _name in dir(__ssl__):
-    if _name in __implements__:
-        continue
-    if _name.startswith('__'):
-        continue
-    if _name == 'socket':
+__imports__ = copy_globals(__ssl__, globals(),
                            # SSLSocket *must* subclass gevent.socket.socket; see issue 597
-        continue
-    _value = getattr(__ssl__, _name)
-    globals()[_name] = _value
-    __imports__.append(_name)
-
-del _name, _value
+                           names_to_ignore=__implements__ + ['socket', 'namedtuple'],
+                           dunder_names_to_keep=())
 
 try:
     _delegate_methods

gevent/_util.py

+# -*- coding: utf-8 -*-
+"""
+internal gevent utilities, not for external use.
+"""
+
+from __future__ import print_function, absolute_import, division
+
+from gevent._compat import iteritems
+
+
+class _NONE(object):
+    """
+    A special object you must never pass to any gevent API.
+    Used as a marker object for keyword arguments that cannot have the
+    builtin None (because that might be a valid value).
+    """
+    __slots__ = ()
+
+    def __repr__(self):
+        return '<default value>'
+
+_NONE = _NONE()
+
+def copy_globals(source,
+                 globs,
+                 only_names=None,
+                 ignore_missing_names=False,
+                 names_to_ignore=(),
+                 dunder_names_to_keep=('__implements__', '__all__', '__imports__'),
+                 cleanup_globs=True):
+    """
+    Copy attributes defined in `source.__dict__` to the dictionary in globs
+    (which should be the caller's globals()).
+
+    Names that start with `__` are ignored (unless they are in
+    *dunder_names_to_keep*). Anything found in *names_to_ignore* is
+    also ignored.
+
+    If *only_names* is given, only those attributes will be considered.
+    In this case, *ignore_missing_names* says whether or not to raise an AttributeError
+    if one of those names can't be found.
+
+    If cleanup_globs has a true value, then common things imported but not used
+    at runtime are removed, including this function.
+
+    Returns a list of the names copied
+    """
+    if only_names:
+        if ignore_missing_names:
+            items = ((k, getattr(source, k, _NONE)) for k in only_names)
+        else:
+            items = ((k, getattr(source, k)) for k in only_names)
+    else:
+        items = iteritems(source.__dict__)
+
+    copied = []
+    for key, value in items:
+        if value is _NONE:
+            continue
+        if key in names_to_ignore:
+            continue
+        if key.startswith("__") and key not in dunder_names_to_keep:
+            continue
+        globs[key] = value
+        copied.append(key)
+
+    if cleanup_globs:
+        if 'copy_globals' in globs:
+            del globs['copy_globals']
+
+    return copied

gevent/core.py

@@ -3,6 +3,8 @@ from __future__ import absolute_import
 
 import os
 
+from gevent._util import copy_globals
+
 try:
     if os.environ.get('GEVENT_CORE_CFFI_ONLY'):
         raise ImportError("Not attempting corecext")
@@ -15,10 +17,6 @@ except ImportError:
     # CFFI/PyPy
     from gevent import corecffi as _core
 
-for item in dir(_core):
-    if item.startswith('__'):
-        continue
-    globals()[item] = getattr(_core, item)
-
+copy_globals(_core, globals())
 
 __all__ = _core.__all__

gevent/hub.py

@@ -28,6 +28,7 @@ __all__ = [
 
 from gevent._compat import string_types
 from gevent._compat import xrange
+from gevent._util import _NONE
 
 if sys.version_info[0] <= 2:
     import thread # pylint:disable=import-error
@@ -57,14 +58,6 @@ get_ident = thread.get_ident
 MAIN_THREAD = get_ident()
 
 
-class _NONE(object):
-    "A special thingy you must never pass to any of gevent API"
-    __slots__ = ()
-
-    def __repr__(self):
-        return '<_NONE>'
-
-_NONE = _NONE()
 
 
 class LoopExit(Exception):

gevent/socket.py

@@ -15,6 +15,7 @@ as well as the constants from the :mod:`socket` module are imported into this mo
 
 import sys
 from gevent._compat import PY3
+from gevent._util import copy_globals
 
 
 if PY3:
@@ -35,10 +36,10 @@ def getfqdn(*args):
     # pylint:disable=unused-argument
     raise NotImplementedError()
 
-for key in _source.__dict__:
-    if key.startswith('__') and key not in '__implements__ __dns__ __all__ __extensions__ __imports__ __socket__'.split():
-        continue
-    globals()[key] = getattr(_source, key)
+copy_globals(_source, globals(),
+             dunder_names_to_keep=('__implements__', '__dns__', '__all__',
+                                   '__extensions__', '__imports__', '__socket__'),
+             cleanup_globs=False)
 
 # The _socket2 and _socket3 don't import things defined in
 # __extensions__, to help avoid confusing reference cycles in the
@@ -49,10 +50,8 @@ for key in _source.__dict__:
 #             module gevent._socket2, attribute cancel_wait
 # These can be ignored.)
 from gevent import _socketcommon
-for key in _socketcommon.__extensions__:
-    globals()[key] = getattr(_socketcommon, key)
-
-del key
+copy_globals(_socketcommon, globals(),
+             only_names=_socketcommon.__extensions__)
 
 try:
     _GLOBAL_DEFAULT_TIMEOUT = __socket__._GLOBAL_DEFAULT_TIMEOUT

gevent/ssl.py

@@ -2,7 +2,7 @@
 Secure Sockets Layer (SSL/TLS) module.
 """
 from gevent._compat import PY2
-
+from gevent._util import copy_globals
 
 if PY2:
     if hasattr(__import__('ssl'), 'SSLContext'):
@@ -18,7 +18,4 @@ else:
     from gevent import _ssl3 as _source # pragma: no cover
 
 
-for key in _source.__dict__:
-    if key.startswith('__') and key not in ('__implements__', '__all__', ' __imports__'):
-        continue
-    globals()[key] = getattr(_source, key)
+copy_globals(_source, globals())

gevent/subprocess.py

@@ -42,6 +42,8 @@ from gevent.hub import get_hub, linkproxy, sleep, getcurrent
 from gevent._compat import integer_types, string_types, xrange
 from gevent._compat import PY3
 from gevent._compat import reraise
+from gevent._util import _NONE
+from gevent._util import copy_globals
 from gevent.fileobject import FileObject
 from gevent.greenlet import Greenlet, joinall
 spawn = Greenlet.spawn
@@ -127,22 +129,18 @@ if sys.version_info[:2] >= (3, 5):
     except:
         MAXFD = 256
 
+actually_imported = copy_globals(__subprocess__, globals(),
+                                 only_names=__imports__,
+                                 ignore_missing_names=True)
+# anything we couldn't import from here we may need to find
+# elsewhere
+__extra__.extend(set(__imports__).difference(set(actually_imported)))
+__imports__ = actually_imported
+del actually_imported
 
-for name in __imports__[:]:
-    try:
-        value = getattr(__subprocess__, name)
-        globals()[name] = value
-    except AttributeError:
-        __imports__.remove(name)
-        __extra__.append(name)
-
-if sys.version_info[:2] <= (2, 6):
-    __implements__.remove('check_output')
-    __extra__.append('check_output')
 
 # In Python 3 on Windows, a lot of the functions previously
 # in _subprocess moved to _winapi
-_NONE = object()
 _subprocess = getattr(__subprocess__, '_subprocess', _NONE)
 _winapi = getattr(__subprocess__, '_winapi', _NONE)
 

gevent/thread.py

@@ -31,6 +31,7 @@ else:
                     'start_new']
 error = __thread__.error
 from gevent._compat import PY3
+from gevent._util import copy_globals
 from gevent.hub import getcurrent, GreenletExit
 from gevent.greenlet import Greenlet
 from gevent.lock import BoundedSemaphore
@@ -98,12 +99,9 @@ if hasattr(__thread__, 'stack_size'):
 else:
     __implements__.remove('stack_size')
 
-for name in __imports__[:]:
-    try:
-        value = getattr(__thread__, name)
-        globals()[name] = value
-    except AttributeError:
-        __imports__.remove(name)
+__imports__ = copy_globals(__thread__, globals(),
+                           only_names=__imports__,
+                           ignore_missing_names=True)
 
 __all__ = __implements__ + __imports__
 __all__.remove('_local')

greentest/test__all__.py

@@ -206,9 +206,14 @@ are missing from %r:
         self.check_completeness()
 
     for path, modname in walk_modules(include_so=True):
+        orig_modname = modname
         modname = modname.replace('gevent.', '').split('.')[0]
+        if not modname:
+            print("WARNING: No such module '%s' at '%s'" % (orig_modname, path),
+                  file=sys.stderr)
+            continue
         exec('''def test_%s(self): self._test("gevent.%s")''' % (modname, modname))
-    del path, modname
+    del path, modname, orig_modname
 
 
 if __name__ == "__main__":