Commit 2ef0a109 authored by Jason Madden's avatar Jason Madden

Fix SSL tests on Python 3.8.

Fix pylint 2.4 new warnings.

Attempting to match Python 3.8 fixes in subprocess.py, especially on Windows.

Attempting to fix psutil-related regression in PyPy 7.2
parent 9796e85a
......@@ -49,6 +49,9 @@ limit-inference-results=1
# @property
# def foo(self):
# return None # generates useless-return
# Pylint 2.4 adds import-outside-toplevel. But we do that a lot to defer imports because of patching.
# Pylint 2.4 adds self-assigning-variable. But we do *that* to avoid unused-import when we
# "export" the variable and don't have a __all__.
disable=wrong-import-position,
wrong-import-order,
missing-docstring,
......@@ -69,7 +72,9 @@ disable=wrong-import-position,
undefined-all-variable,
inconsistent-return-statements,
useless-return,
useless-object-inheritance
useless-object-inheritance,
import-outside-toplevel,
self-assigning-variable
[FORMAT]
......
......@@ -31,29 +31,29 @@ __version__ = '1.5a3.dev0'
__all__ = [
'get_hub',
'Greenlet',
'GreenletExit',
'spawn',
'spawn_later',
'spawn_raw',
'iwait',
'wait',
'killall',
'Timeout',
'with_timeout',
'config', # Added in 1.3a2
'fork',
'get_hub',
'getcurrent',
'sleep',
'getswitchinterval',
'idle',
'iwait',
'joinall',
'kill',
'signal', # deprecated
'signal_handler',
'fork',
'killall',
'reinit',
'getswitchinterval',
'setswitchinterval',
# Added in 1.3a2
'config',
'signal', # deprecated
'signal_handler',
'sleep',
'spawn',
'spawn_later',
'spawn_raw',
'wait',
'with_timeout',
]
......@@ -89,11 +89,9 @@ from gevent._hub_primitives import iwait_on_objects as iwait
from gevent._hub_primitives import wait_on_objects as wait
from gevent.greenlet import Greenlet, joinall, killall
joinall = joinall # export for pylint
spawn = Greenlet.spawn
spawn_later = Greenlet.spawn_later
#: The singleton configuration object for gevent.
config = config
from gevent.timeout import Timeout, with_timeout
from gevent.hub import getcurrent, GreenletExit, spawn_raw, sleep, idle, kill, reinit
......@@ -163,7 +161,7 @@ del sys
# outdated on each major release.
def __dependencies_for_freezing(): # pragma: no cover
# pylint:disable=unused-import
# pylint:disable=unused-import, import-outside-toplevel
from gevent import core
from gevent import resolver_thread
from gevent import resolver_ares
......
......@@ -47,6 +47,14 @@ def NativeStrIO():
import io
return io.BytesIO() if str is bytes else io.StringIO()
try:
from abc import ABC
except ImportError:
import abc
ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()})
del abc
## Exceptions
if PY3:
def reraise(t, value, tb=None): # pylint:disable=unused-argument
......@@ -87,6 +95,16 @@ else:
from itertools import izip # python 3: pylint:disable=no-member,no-name-in-module
izip = izip
## The __fspath__ protocol
try:
from os import PathLike # pylint:disable=unused-import
except ImportError:
class PathLike(ABC):
@classmethod
def __subclasshook__(cls, subclass):
return hasattr(subclass, '__fspath__')
# fspath from 3.6 os.py, but modified to raise the same exceptions as the
# real native implementation.
# Define for testing
......@@ -152,6 +170,19 @@ except ImportError:
# Not sure how to handle this.
raise UnicodeEncodeError("Can't encode path to filesystem encoding")
try:
from os import fsdecode # pylint:disable=unused-import
except ImportError:
def fsdecode(filename):
"""Decode filename (an os.PathLike, bytes, or str) from the filesystem
encoding with 'surrogateescape' error handler, return str unchanged. On
Windows, use 'strict' error handler if the file system encoding is
'mbcs' (which is the default encoding).
"""
filename = fspath(filename) # Does type-checking of `filename`.
if PY3 and isinstance(filename, bytes):
return filename.decode(encoding, errors)
return filename
## Clocks
try:
......@@ -162,7 +193,7 @@ except ImportError:
import time
if sys.platform == "win32":
perf_counter = time.clock
perf_counter = time.clock # pylint:disable=no-member
else:
perf_counter = time.time
......
......@@ -206,8 +206,7 @@ class IMap(IMapUnordered):
index, value = self.queue.get()
if index == self.index:
break
else:
self._results[index] = value
self._results[index] = value
self.index += 1
return value
......
......@@ -261,7 +261,7 @@ class socket(object):
result = self._sock.connect_ex(address)
if not result or result == EISCONN:
break
elif (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows):
if (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows):
self._wait(self._write_event)
else:
raise error(result, strerror(result))
......
......@@ -398,7 +398,7 @@ class socket(object):
if not result or result == EISCONN:
break
elif (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows):
if (result in (EWOULDBLOCK, EINPROGRESS, EALREADY)) or (result == EINVAL and is_windows):
self._wait(self._write_event)
else:
if (isinstance(address, tuple)
......
......@@ -235,8 +235,7 @@ class SSLSocket(socket):
sys.exc_clear()
self._wait(self._read_event)
continue
else:
raise
raise
else:
return socket.recv_into(self, buffer, nbytes, flags)
......
......@@ -44,6 +44,31 @@ if 'namedtuple' in __all__:
orig_SSLContext = __ssl__.SSLContext # pylint:disable=no-member
# We have to pass the raw stdlib socket to SSLContext.wrap_socket.
# That method in turn can pass that object on to things like SNI callbacks.
# It wouldn't have access to any of the attributes on the SSLSocket, like
# context, that it's supposed to (see test_ssl.test_sni_callback). Previously
# we just delegated to the sslsocket with __getattr__, but 3.8
# added some new callbacks and a test that the object they get is an instance
# of the high-level SSLSocket class, so that doesn't work anymore. Instead,
# we wrap the callback and get the real socket to pass on.
class _contextawaresock(socket._gevent_sock_class):
__slots__ = ('_sslsock',)
def __init__(self, family, type, proto, fileno, sslsocket_wref):
super().__init__(family, type, proto, fileno)
self._sslsock = sslsocket_wref
class _Callback(object):
__slots__ = ('user_function',)
def __init__(self, user_function):
self.user_function = user_function
def __call__(self, conn, *args):
conn = conn._sslsock()
return self.user_function(conn, *args)
class SSLContext(orig_SSLContext):
......@@ -68,10 +93,6 @@ class SSLContext(orig_SSLContext):
_context=self,
_session=session)
if not hasattr(orig_SSLContext, 'check_hostname'):
# Python 3.3 lacks this
check_hostname = False
if hasattr(orig_SSLContext.options, 'setter'):
# In 3.6, these became properties. They want to access the
# property __set__ method in the superclass, and they do so by using
......@@ -108,52 +129,44 @@ class SSLContext(orig_SSLContext):
# SSLContext back. This function cannot switch, so it should be safe,
# unless somehow we have multiple threads in a monkey-patched ssl module
# at the same time, which doesn't make much sense.
@orig_SSLContext._msg_callback.setter
@property
def _msg_callback(self):
result = super()._msg_callback
if isinstance(result, _Callback):
result = result.user_function
return result
@_msg_callback.setter
def _msg_callback(self, value):
if value and callable(value):
value = _Callback(value)
__ssl__.SSLContext = orig_SSLContext
try:
super(SSLContext, SSLContext)._msg_callback.__set__(self, value)
finally:
__ssl__.SSLContext = SSLContext
class _contextawaresock(socket._gevent_sock_class):
# We have to pass the raw stdlib socket to SSLContext.wrap_socket.
# That method in turn can pass that object on to things like SNI callbacks.
# It wouldn't have access to any of the attributes on the SSLSocket, like
# context, that it's supposed to (see test_ssl.test_sni_callback). Our
# solution is to keep a weak reference to the SSLSocket on the raw
# socket and delegate.
# We keep it in a slot to avoid having the ability to set any attributes
# we're not prepared for (because we don't know what to delegate.)
__slots__ = ('_sslsock',)
@property
def context(self):
return self._sslsock().context
@context.setter
def context(self, ctx):
self._sslsock().context = ctx
@property
def session(self):
"""The SSLSession for client socket."""
return self._sslsock().session
if hasattr(orig_SSLContext, 'sni_callback'):
# Added in 3.7.
@property
def sni_callback(self):
result = super().sni_callback
if isinstance(result, _Callback):
result = result.user_function
return result
@sni_callback.setter
def sni_callback(self, value):
if value and callable(value):
value = _Callback(value)
super(orig_SSLContext, orig_SSLContext).sni_callback.__set__(self, value)
else:
# In newer versions, this just sets sni_callback.
def set_servername_callback(self, cb):
if cb and callable(cb):
cb = _Callback(cb)
super().set_servername_callback(cb)
@session.setter
def session(self, session):
self._sslsock().session = session
def __getattr__(self, name):
try:
return getattr(self._sslsock(), name)
except RuntimeError:
# XXX: If the attribute doesn't exist,
# we infinitely recurse
pass
raise AttributeError(name)
class SSLSocket(socket):
"""
......@@ -164,8 +177,6 @@ class SSLSocket(socket):
# pylint:disable=too-many-instance-attributes,too-many-public-methods
_gevent_sock_class = _contextawaresock
def __init__(self, sock=None, keyfile=None, certfile=None,
server_side=False, cert_reqs=CERT_NONE,
ssl_version=PROTOCOL_SSLv23, ca_certs=None,
......@@ -243,7 +254,6 @@ class SSLSocket(socket):
else:
socket.__init__(self, family=family, type=type, proto=proto)
self._sock._sslsock = _wref(self)
self._closed = False
self._sslobj = None
# see if we're connected
......@@ -274,6 +284,9 @@ class SSLSocket(socket):
self.close()
raise x
def _gevent_sock_class(self, family, type, proto, fileno):
return _contextawaresock(family, type, proto, fileno, _wref(self))
def _extra_repr(self):
return ' server=%s, cipher=%r' % (
self.server_side,
......
......@@ -462,18 +462,29 @@ class Hub(WaitOperationsGreenlet):
def handle_error(self, context, type, value, tb):
"""
Called by the event loop when an error occurs. The arguments
type, value, and tb are the standard tuple returned by :func:`sys.exc_info`.
Called by the event loop when an error occurs. The default
action is to print the exception to the :attr:`exception
stream <exception_stream>`.
Applications can set a property on the hub with this same signature
to override the error handling provided by this class.
The arguments ``type``, ``value``, and ``tb`` are the standard
tuple as returned by :func:`sys.exc_info`. (Note that when
this is called, it may not be safe to call
:func:`sys.exc_info`.)
Errors that are :attr:`system errors <SYSTEM_ERROR>` are passed
to :meth:`handle_system_error`.
Errors that are :attr:`not errors <NOT_ERROR>` are not
printed.
:param context: If this is ``None``, indicates a system error that
should generally result in exiting the loop and being thrown to the
parent greenlet.
Errors that are :attr:`system errors <SYSTEM_ERROR>` are
passed to :meth:`handle_system_error` after being printed.
Applications can set a property on the hub instance with this
same signature to override the error handling provided by this
class. This is an advanced usage and requires great care. This
function *must not* raise any exceptions.
:param context: If this is ``None``, indicates a system error
that should generally result in exiting the loop and being
thrown to the parent greenlet.
"""
if isinstance(value, str):
# Cython can raise errors where the value is a plain string
......@@ -513,7 +524,8 @@ class Hub(WaitOperationsGreenlet):
def exception_stream(self):
"""
The stream to which exceptions will be written.
Defaults to ``sys.stderr`` unless assigned to.
Defaults to ``sys.stderr`` unless assigned. Assigning a
false (None) value disables printing exceptions.
.. versionadded:: 1.2a1
"""
......
......@@ -49,8 +49,9 @@ from gevent._compat import PY36
from gevent._compat import PY37
from gevent._compat import PY38
from gevent._compat import reraise
from gevent._compat import fspath
from gevent._compat import fsdecode
from gevent._compat import fsencode
from gevent._compat import PathLike
from gevent._util import _NONE
from gevent._util import copy_globals
......@@ -648,7 +649,7 @@ class Popen(object):
# Convert here for the sake of all platforms. os.chdir accepts
# path-like objects natively under 3.6, but CreateProcess
# doesn't.
cwd = fspath(cwd) if cwd is not None else None
cwd = fsdecode(cwd) if cwd is not None else None
try:
self._execute_child(args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
......@@ -983,6 +984,22 @@ class Popen(object):
"""Execute program (MS Windows version)"""
# pylint:disable=undefined-variable
assert not pass_fds, "pass_fds not supported on Windows."
if isinstance(args, str):
pass
elif isinstance(args, bytes):
if shell and PY3:
raise TypeError('bytes args is not allowed on Windows')
args = list2cmdline([args])
elif isinstance(args, PathLike):
if shell:
raise TypeError('path-like args is not allowed when '
'shell is true')
args = list2cmdline([args])
else:
args = list2cmdline(args)
if executable is not None:
executable = fsdecode(executable)
if not isinstance(args, string_types):
args = list2cmdline(args)
......@@ -1060,7 +1077,7 @@ class Popen(object):
int(not close_fds),
creationflags,
env,
cwd,
cwd, # fsdecode handled earlier
startupinfo)
except IOError as e: # From 2.6 on, pywintypes.error was defined as IOError
# Translate pywintypes.error to WindowsError, which is
......@@ -1345,14 +1362,20 @@ class Popen(object):
args = [args]
elif not PY3 and isinstance(args, string_types):
args = [args]
elif isinstance(args, PathLike):
if shell:
raise TypeError('path-like args is not allowed when '
'shell is true')
args = [fsencode(args)] # os.PathLike -> [str]
else:
try:
args = list(args)
except TypeError: # os.PathLike instead of a sequence?
args = [fsencode(args)] # os.PathLike -> [str]
args = list(args)
if shell:
args = ["/bin/sh", "-c"] + args
# On Android the default shell is at '/system/bin/sh'.
unix_shell = (
'/system/bin/sh' if hasattr(sys, 'getandroidapilevel') else '/bin/sh'
)
args = [unix_shell, "-c"] + args
if executable:
args[0] = executable
......
......@@ -24,6 +24,7 @@ from .patched_tests_setup import disable_tests_in_source
from . import support
from . import resources
from . import SkipTest
from . import util
if RUNNING_ON_APPVEYOR and PY37:
# 3.7 added a stricter mode for thread cleanup.
......@@ -42,6 +43,15 @@ if RUNNING_ON_APPVEYOR and PY37:
# Configure allowed resources
resources.setup_resources()
if not os.path.exists(test_filename) and os.sep not in test_filename:
# A simple filename, given without a path, that doesn't exist.
# So we change to the appropriate directory, if we can find it.
# This happens when copy-pasting the output of the testrunner
for d in util.find_stdlib_tests():
if os.path.exists(os.path.join(d, test_filename)):
os.chdir(d)
break
__file__ = os.path.join(os.getcwd(), test_filename)
test_name = os.path.splitext(test_filename)[0]
......
......@@ -108,6 +108,17 @@ else:
(socket.listen(1)). Unlike the lsof implementation, this will only
return sockets in a state like that.
"""
if sysinfo.PYPY:
# We've seen OSError: No such file or directory /proc/PID/fd/NUM.
# This occurs in the loop that checks open files. It first does listdir()
# and then tries readlink() on each file. But the file went away.
# This must be because of async GC in PyPy running destructors at arbitrary
# times. This became an issue in PyPy 7.2. Try to clean that up before we
# begin.
import gc
gc.collect()
gc.collect()
results = dict()
process = psutil.Process()
results['data'] = process.open_files() + process.connections('all')
......
......@@ -137,3 +137,16 @@ RESOLVER_ARES = os.getenv('GEVENT_RESOLVER') == 'ares'
RESOLVER_DNSPYTHON = os.getenv('GEVENT_RESOLVER') == 'dnspython'
RESOLVER_NOT_SYSTEM = RESOLVER_ARES or RESOLVER_DNSPYTHON
def get_python_version():
"""
Return a string of the simple python version,
such as '3.8.0b4'. Handles alpha and beta and final releases.
"""
version = '%s.%s.%s' % sys.version_info[:3]
if sys.version_info[3] == 'alpha':
version += 'a%s' % sys.version_info[4]
elif sys.version_info[3] == 'beta':
version += 'b%s' % sys.version_info[4]
return version
......@@ -418,6 +418,40 @@ def search_for_setup_py(a_file=None, a_module_name=None, a_class=None, climb_cwd
raise NoSetupPyFound("After checking %r" % (locals(),))
def _version_dir_components():
directory = '%s.%s' % sys.version_info[:2]
full_directory = '%s.%s.%s' % sys.version_info[:3]
if hasattr(sys, 'pypy_version_info'):
directory += 'pypy'
full_directory += 'pypy'
return directory, full_directory
def find_stdlib_tests():
"""
Return a sequence of directories that could contain
stdlib tests for the running version of Python.
The most specific tests are at the end of the sequence.
No checks are performed on existence of the directories.
"""
setup_py = search_for_setup_py(a_file=__file__)
greentest = os.path.join(setup_py, 'src', 'greentest')
directory, full_directory = _version_dir_components()
directory = '%s.%s' % sys.version_info[:2]
full_directory = '%s.%s.%s' % sys.version_info[:3]
if hasattr(sys, 'pypy_version_info'):
directory += 'pypy'
full_directory += 'pypy'
directory = os.path.join(greentest, directory)
full_directory = os.path.join(greentest, full_directory)
return directory, full_directory
class ExampleMixin(object):
"Something that uses the examples/ directory"
......
......@@ -6,35 +6,12 @@ import atexit
# subprocess: include in subprocess tests
from gevent.testing import util
from gevent.testing import sysinfo
TIMEOUT = 120
# XXX: Generalize this so other packages can use it.
def find_stdlib_tests():
setup_py = util.search_for_setup_py(a_file=__file__)
greentest = os.path.join(setup_py, 'src', 'greentest')
directory = '%s.%s' % sys.version_info[:2]
full_directory = '%s.%s.%s' % sys.version_info[:3]
if hasattr(sys, 'pypy_version_info'):
directory += 'pypy'
full_directory += 'pypy'
directory = os.path.join(greentest, directory)
full_directory = os.path.join(greentest, full_directory)
return directory, full_directory
def get_python_version():
version = '%s.%s.%s' % sys.version_info[:3]
if sys.version_info[3] == 'alpha':
version += 'a%s' % sys.version_info[4]
elif sys.version_info[3] == 'beta':
version += 'b%s' % sys.version_info[4]
return version
def get_absolute_pythonpath():
paths = [os.path.abspath(p) for p in os.environ.get('PYTHONPATH', '').split(os.pathsep)]
......@@ -43,7 +20,7 @@ def get_absolute_pythonpath():
def TESTRUNNER(tests=None):
try:
test_dir, version_test_dir = find_stdlib_tests()
test_dir, version_test_dir = util.find_stdlib_tests()
except util.NoSetupPyFound as e:
util.log("WARNING: No setup.py and src/greentest found: %r", e,
color="suboptimal-behaviour")
......@@ -57,7 +34,7 @@ def TESTRUNNER(tests=None):
with open(os.path.join(test_dir, 'version')) as f:
preferred_version = f.read().strip()
running_version = get_python_version()
running_version = sysinfo.get_python_version()
if preferred_version != running_version:
util.log('WARNING: The tests in %s/ are from version %s and your Python is %s',
test_dir, preferred_version, running_version,
......
import errno
import os
import sys
#os.environ['GEVENT_NOWAITPID'] = 'True'
import gevent
import gevent.monkey
......@@ -19,8 +18,27 @@ def handle_sigchld(*_args):
# Raise an ignored error
raise TypeError("This should be ignored but printed")
# Try to produce output compatible with unittest output so
# our status parsing functions work.
import signal
if hasattr(signal, 'SIGCHLD'):
# In Python 3.8.0 final, on both Travis CI/Linux and locally
# on macOS, the *child* process started crashing on exit with a memory
# error:
#
# Debug memory block at address p=0x7fcf5d6b5000: API ''
# 6508921152173528397 bytes originally requested
# The 7 pad bytes at p-7 are not all FORBIDDENBYTE (0xfd):
#
# When PYTHONDEVMODE is set. This happens even if we just simply fork
# the child process and don't have gevent even /imported/ in the most
# minimal test case. It's not clear what caused that.
if sys.version_info[:2] >= (3, 8) and os.environ.get("PYTHONDEVMODE"):
print("Ran 1 tests in 0.0s (skipped=1)")
sys.exit(0)
assert signal.getsignal(signal.SIGCHLD) == signal.SIG_DFL
signal.signal(signal.SIGCHLD, handle_sigchld)
handler = signal.getsignal(signal.SIGCHLD)
......@@ -64,6 +82,8 @@ if hasattr(signal, 'SIGCHLD'):
raise AssertionError("Failed to wait using", func)
finally:
timeout.close()
print("Ran 1 tests in 0.0s")
sys.exit(0)
else:
print("No SIGCHLD, not testing")
print("Ran 1 tests in 0.0s (skipped=1)")
......@@ -636,11 +636,11 @@ class TestChunkedPost(TestCase):
return [data]
if env['PATH_INFO'] == '/b':
lines = [x for x in iter(lambda: env['wsgi.input'].read(6), b'')]
lines = list(iter(lambda: env['wsgi.input'].read(6), b''))
return lines
if env['PATH_INFO'] == '/c':
return [x for x in iter(lambda: env['wsgi.input'].read(1), b'')]
return list(iter(lambda: env['wsgi.input'].read(1), b''))
def test_014_chunked_post(self):
data = (b'POST /a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment