Merge pull request #398 from surfly/socket_python3_r2

initial impl for socket/ssl for python3 #38

examples/concurrent_download.py

@@ -3,6 +3,7 @@
 
 """Spawn multiple workers and wait for them to complete"""
 from __future__ import print_function
+import sys
 
 urls = ['http://www.google.com', 'http://www.yandex.ru', 'http://www.python.org']
 
@@ -12,12 +13,16 @@ from gevent import monkey
 # patches stdlib (including socket and ssl modules) to cooperate with other greenlets
 monkey.patch_all()
 
-import urllib2
+
+if sys.version_info[0] == 3:
+    from urllib.request import urlopen
+else:
+    from urllib2 import urlopen
 
 
 def print_head(url):
     print('Starting %s' % url)
-    data = urllib2.urlopen(url).read()
+    data = urlopen(url).read()
     print('%s: %s bytes: %r' % (url, len(data), data[:50]))
 
 jobs = [gevent.spawn(print_head, url) for url in urls]

gevent/_socketcommon.py

+# Copyright (c) 2009-2014 Denis Bilenko and gevent contributors. See LICENSE for details.
+from __future__ import absolute_import
+
+# standard functions and classes that this module re-implements in a gevent-aware way:
+_implements = ['create_connection',
+               'socket',
+               'SocketType',
+               'fromfd',
+               'socketpair']
+
+__dns__ = ['getaddrinfo',
+           'gethostbyname',
+           'gethostbyname_ex',
+           'gethostbyaddr',
+           'getnameinfo',
+           'getfqdn']
+
+_implements += __dns__
+
+# non-standard functions that this module provides:
+__extensions__ = ['wait_read',
+                  'wait_write',
+                  'wait_readwrite']
+
+# standard functions and classes that this module re-imports
+__imports__ = ['error',
+               'gaierror',
+               'herror',
+               'htonl',
+               'htons',
+               'ntohl',
+               'ntohs',
+               'inet_aton',
+               'inet_ntoa',
+               'inet_pton',
+               'inet_ntop',
+               'timeout',
+               'gethostname',
+               'getprotobyname',
+               'getservbyname',
+               'getservbyport',
+               'getdefaulttimeout',
+               'setdefaulttimeout',
+               # Windows:
+               'errorTab']
+
+
+import sys
+from gevent.hub import get_hub, string_types, integer_types
+from gevent.timeout import Timeout
+
+is_windows = sys.platform == 'win32'
+
+if is_windows:
+    # no such thing as WSAEPERM or error code 10001 according to winsock.h or MSDN
+    from errno import WSAEINVAL as EINVAL
+    from errno import WSAEWOULDBLOCK as EWOULDBLOCK
+    from errno import WSAEINPROGRESS as EINPROGRESS
+    from errno import WSAEALREADY as EALREADY
+    from errno import WSAEISCONN as EISCONN
+    from gevent.win32util import formatError as strerror
+    EAGAIN = EWOULDBLOCK
+else:
+    from errno import EINVAL
+    from errno import EWOULDBLOCK
+    from errno import EINPROGRESS
+    from errno import EALREADY
+    from errno import EAGAIN
+    from errno import EISCONN
+    from os import strerror
+
+try:
+    from errno import EBADF
+except ImportError:
+    EBADF = 9
+
+import _socket
+_realsocket = _socket.socket
+import socket as __socket__
+
+
+for name in __imports__[:]:
+    try:
+        value = getattr(__socket__, name)
+        globals()[name] = value
+    except AttributeError:
+        __imports__.remove(name)
+
+for name in __socket__.__all__:
+    value = getattr(__socket__, name)
+    if isinstance(value, integer_types) or isinstance(value, string_types):
+        globals()[name] = value
+        __imports__.append(name)
+
+del name, value
+
+
+def wait(io, timeout=None, timeout_exc=timeout('timed out')):
+    """Block the current greenlet until *io* is ready.
+
+    If *timeout* is non-negative, then *timeout_exc* is raised after *timeout* second has passed.
+    By default *timeout_exc* is ``socket.timeout('timed out')``.
+
+    If :func:`cancel_wait` is called, raise ``socket.error(EBADF, 'File descriptor was closed in another greenlet')``.
+    """
+    assert io.callback is None, 'This socket is already used by another greenlet: %r' % (io.callback, )
+    if timeout is not None:
+        timeout = Timeout.start_new(timeout, timeout_exc)
+    try:
+        return get_hub().wait(io)
+    finally:
+        if timeout is not None:
+            timeout.cancel()
+    # rename "io" to "watcher" because wait() works with any watcher
+
+
+def wait_read(fileno, timeout=None, timeout_exc=timeout('timed out')):
+    """Block the current greenlet until *fileno* is ready to read.
+
+    If *timeout* is non-negative, then *timeout_exc* is raised after *timeout* second has passed.
+    By default *timeout_exc* is ``socket.timeout('timed out')``.
+
+    If :func:`cancel_wait` is called, raise ``socket.error(EBADF, 'File descriptor was closed in another greenlet')``.
+    """
+    io = get_hub().loop.io(fileno, 1)
+    return wait(io, timeout, timeout_exc)
+
+
+def wait_write(fileno, timeout=None, timeout_exc=timeout('timed out'), event=None):
+    """Block the current greenlet until *fileno* is ready to write.
+
+    If *timeout* is non-negative, then *timeout_exc* is raised after *timeout* second has passed.
+    By default *timeout_exc* is ``socket.timeout('timed out')``.
+
+    If :func:`cancel_wait` is called, raise ``socket.error(EBADF, 'File descriptor was closed in another greenlet')``.
+    """
+    io = get_hub().loop.io(fileno, 2)
+    return wait(io, timeout, timeout_exc)
+
+
+def wait_readwrite(fileno, timeout=None, timeout_exc=timeout('timed out'), event=None):
+    """Block the current greenlet until *fileno* is ready to read or write.
+
+    If *timeout* is non-negative, then *timeout_exc* is raised after *timeout* second has passed.
+    By default *timeout_exc* is ``socket.timeout('timed out')``.
+
+    If :func:`cancel_wait` is called, raise ``socket.error(EBADF, 'File descriptor was closed in another greenlet')``.
+    """
+    io = get_hub().loop.io(fileno, 3)
+    return wait(io, timeout, timeout_exc)
+
+
+cancel_wait_ex = error(EBADF, 'File descriptor was closed in another greenlet')
+
+
+def cancel_wait(watcher):
+    get_hub().cancel_wait(watcher, cancel_wait_ex)
+
+
+class BlockingResolver(object):
+
+    def __init__(self, hub=None):
+        pass
+
+    def close(self):
+        pass
+
+    for method in ['gethostbyname',
+                   'gethostbyname_ex',
+                   'getaddrinfo',
+                   'gethostbyaddr',
+                   'getnameinfo']:
+        locals()[method] = staticmethod(getattr(_socket, method))
+
+
+def gethostbyname(hostname):
+    return get_hub().resolver.gethostbyname(hostname)
+
+
+def gethostbyname_ex(hostname):
+    return get_hub().resolver.gethostbyname_ex(hostname)
+
+
+def getaddrinfo(host, port, family=0, socktype=0, proto=0, flags=0):
+    return get_hub().resolver.getaddrinfo(host, port, family, socktype, proto, flags)
+
+
+def gethostbyaddr(ip_address):
+    return get_hub().resolver.gethostbyaddr(ip_address)
+
+
+def getnameinfo(sockaddr, flags):
+    return get_hub().resolver.getnameinfo(sockaddr, flags)
+
+
+def getfqdn(name=''):
+    """Get fully qualified domain name from name.
+
+    An empty argument is interpreted as meaning the local host.
+
+    First the hostname returned by gethostbyaddr() is checked, then
+    possibly existing aliases. In case no FQDN is available, hostname
+    from gethostname() is returned.
+    """
+    name = name.strip()
+    if not name or name == '0.0.0.0':
+        name = gethostname()
+    try:
+        hostname, aliases, ipaddrs = gethostbyaddr(name)
+    except error:
+        pass
+    else:
+        aliases.insert(0, hostname)
+        for name in aliases:
+            if '.' in name:
+                break
+        else:
+            name = hostname
+    return name

greentest/test__all__.py

@@ -33,7 +33,7 @@ NOT_IMPLEMENTED = {
 COULD_BE_MISSING = {
     'socket': ['create_connection', 'RAND_add', 'RAND_egd', 'RAND_status']}
 
-NO_ALL = ['gevent.threading', 'gevent._util']
+NO_ALL = ['gevent.threading', 'gevent._util', 'gevent._socketcommon']
 
 
 class Test(unittest.TestCase):
@@ -140,6 +140,10 @@ are missing from %r:
             raise AssertionError(msg)
 
     def _test(self, modname):
+        if modname.endswith('2'):
+            return
+        if modname.endswith('3'):
+            return
         self.modname = modname
         six.exec_("import %s" % modname, {})
         self.module = sys.modules[modname]

greentest/test__execmodules.py

+import sys
 from greentest import walk_modules, BaseTestCase, main
 import six
 
@@ -21,6 +22,10 @@ def make_exec_test(path, module):
 
 
 for path, module in walk_modules():
+    if sys.version_info[0] == 3 and path.endswith('2.py'):
+        continue
+    if sys.version_info[0] == 2 and path.endswith('3.py'):
+        continue
     make_exec_test(path, module)
 
 

greentest/test__socket.py

@@ -147,7 +147,7 @@ class TestTCP(greentest.TestCase):
 
         def accept_once():
             conn, addr = self.listener.accept()
-            fd = conn.makefile()
+            fd = conn.makefile(mode='w')
             fd.write('hello\n')
             fd.close()
 

util/pyflakes.py

@@ -8,8 +8,11 @@ import glob
 
 IGNORED = r'''
 gevent/socket.py:\d+: undefined name
+gevent/_socket[23].py:\d+: undefined name
+gevent/_socketcommon.py:\d+: undefined name
+gevent/_socketcommon.py:\d+: .*imported but unused
 gevent/subprocess.py:\d+: undefined name
-gevent/ssl.py:\d+: undefined name
+gevent/_?ssl[23]?.py:\d+: undefined name
 gevent/__init__.py:\d+:.*imported but unused
 gevent/__init__.py:\d+: redefinition of unused 'signal' from line
 gevent/coros.py:\d+: 'from gevent.lock import *' used; unable to detect undefined names
@@ -68,5 +71,11 @@ def pyflakes(args):
 
 
 pyflakes('examples/ greentest/*.py util/ *.py')
-py = set(glob.glob('gevent/*.py')) - set(['gevent/_util_py2.py'])
+
+if sys.version_info[0] == 3:
+    ignored_files = ['gevent/_util_py2.py', 'gevent/_socket2.py']
+else:
+    ignored_files = ['gevent/_socket3.py']
+
+py = set(glob.glob('gevent/*.py')) - set(ignored_files)
 pyflakes(' '.join(py))