Commit 6766b088 authored by Jason Madden's avatar Jason Madden Committed by GitHub

Merge pull request #1176 from gevent/issue1172

Use /dev/fd|/proc/self/fd to get open FDs to close in Popen
parents b166aaf8 ccc47e33
......@@ -10,6 +10,18 @@
- On Windows, CFFI is now a dependency so that the libuv backend
really can be used by default.
- Fix a bug detecting whether we can use the memory monitoring
features when psutil is not installed.
- `gevent.subprocess.Popen` uses ``/proc/self/fd`` (on Linux) or
``/dev/fd`` (on BSD, including macOS) to find the file descriptors
to close when ``close_fds`` is true. This matches an optimization
added to Python 3 (and backports it to Python 2.7), making process
spawning up to 9 times faster. Also, on Python 3, since Python 3.3
is no longer supported, we can also optimize the case where
``close_fds`` is false (not the default), making process spawning up
to 38 times faster. Initially reported in :issue:`1172` by Ofer Koren.
1.3b1 (2018-04-13)
==================
......
......@@ -179,7 +179,7 @@ develop:
# disables the cache.
# We need wheel>=0.26 on Python 3.5. See previous revisions.
time ${PYTHON} -m pip install -U -r ci-requirements.txt
GEVENTSETUP_EV_VERIFY=3 time ${PYTHON} -m pip install -U -e .
GEVENTSETUP_EV_VERIFY=3 time ${PYTHON} -m pip install -U -e .[test,dnspython,events]
${PYTHON} -m pip freeze
ccache -s
@${PYTHON} scripts/travis.py fold_end install
......
......@@ -142,7 +142,7 @@ build_script:
# into a variable, using python to glob.
- "%PYEXE% -c \"import glob; print(glob.glob('dist/*whl')[0])\" > whl.txt"
- set /p PYWHL=<whl.txt
- "%PYEXE% -m pip install %PYWHL%"
- "%PYEXE% -m pip install %PYWHL%[test,events,dnspython]"
test_script:
# Run the project tests
......
......@@ -12,23 +12,8 @@ greenlet>=0.4.13 ; platform_python_implementation == "CPython"
pylint>=1.8.0
# pyyaml is included here and doesn't install on travis with 3.7a3
prospector[with_pyroma] ; python_version < '3.7'
# We don't run coverage on Windows, and pypy can't build it there
# anyway (coveralls -> cryptopgraphy -> openssl)
coverage>=4.0 ; sys_platform != 'win32'
coveralls>=1.0 ; sys_platform != 'win32'
# See version requirements in setup.py
cffi >= 1.11.5 ; platform_python_implementation == "CPython"
futures
dnspython
idna
# Makes tests faster
# Fails to build on PyPy on Windows.
psutil ; platform_python_implementation == "CPython" or sys_platform != "win32"
# benchmarks use this
perf
# examples, called from tests, use this
requests
# Events
zope.event
zope.interface
......@@ -4,5 +4,4 @@
restview
-r ci-requirements.txt
-r rtd-requirements.txt
-e .
-e .[test,events,dnspython,doc]
......@@ -327,6 +327,27 @@ def run_setup(ext_modules, run_make):
'zope.event',
'zope.interface',
],
'doc': [
'repoze.sphinx.autointerface',
],
'test': [
'zope.interface',
'zope.event',
# Makes tests faster
# Fails to build on PyPy on Windows.
'psutil ; platform_python_implementation == "CPython" or sys_platform != "win32"',
# examples, called from tests, use this
'requests',
# We don't run coverage on Windows, and pypy can't build it there
# anyway (coveralls -> cryptopgraphy -> openssl)
'coverage>=4.0 ; sys_platform != "win32"',
'coveralls>=1.0 ; sys_platform != "win32"',
'futures ; python_version == "2.7"',
'mock ; python_version == "2.7"',
]
},
# It's always safe to pass the CFFI keyword, even if
# cffi is not installed: it's just ignored in that case.
......
......@@ -379,6 +379,12 @@ if 'TimeoutExpired' not in globals():
(self.cmd, self.timeout))
if hasattr(os, 'set_inheritable'):
_set_inheritable = os.set_inheritable
else:
_set_inheritable = lambda i, v: True
class Popen(object):
"""
The underlying process creation and management in this module is
......@@ -1145,42 +1151,75 @@ class Popen(object):
self._set_cloexec_flag(w)
return r, w
def _close_fds(self, keep):
# `keep` is a set of fds, so we
# use os.closerange from 3 to min(keep)
# and then from max(keep + 1) to MAXFD and
# loop through filling in the gaps.
# Under new python versions, we need to explicitly set
# passed fds to be inheritable or they will go away on exec
if hasattr(os, 'set_inheritable'):
set_inheritable = os.set_inheritable
_POSSIBLE_FD_DIRS = (
'/proc/self/fd', # Linux
'/dev/fd', # BSD, including macOS
)
@classmethod
def _close_fds(cls, keep, errpipe_write):
# From the C code:
# errpipe_write is part of keep. It must be closed at
# exec(), but kept open in the child process until exec() is
# called.
for path in cls._POSSIBLE_FD_DIRS:
if os.path.isdir(path):
return cls._close_fds_from_path(path, keep, errpipe_write)
return cls._close_fds_brute_force(keep, errpipe_write)
@classmethod
def _close_fds_from_path(cls, path, keep, errpipe_write):
# path names a directory whose only entries have
# names that are ascii strings of integers in base10,
# corresponding to the fds the current process has open
try:
fds = [int(fname) for fname in os.listdir(path)]
except (ValueError, OSError):
cls._close_fds_brute_force(keep, errpipe_write)
else:
set_inheritable = lambda i, v: True
if hasattr(os, 'closerange'):
keep = sorted(keep)
min_keep = min(keep)
max_keep = max(keep)
os.closerange(3, min_keep)
os.closerange(max_keep + 1, MAXFD)
for i in xrange(min_keep, max_keep):
if i in keep:
set_inheritable(i, True)
for i in keep:
if i == errpipe_write:
continue
_set_inheritable(i, True)
try:
os.close(i)
except:
pass
else:
for i in xrange(3, MAXFD):
if i in keep:
set_inheritable(i, True)
for fd in fds:
if fd in keep or fd < 3:
continue
try:
os.close(i)
os.close(fd)
except:
pass
@classmethod
def _close_fds_brute_force(cls, keep, errpipe_write):
# `keep` is a set of fds, so we
# use os.closerange from 3 to min(keep)
# and then from max(keep + 1) to MAXFD and
# loop through filling in the gaps.
# Under new python versions, we need to explicitly set
# passed fds to be inheritable or they will go away on exec
# XXX: Bug: We implicitly rely on errpipe_write being the largest open
# FD so that we don't change its cloexec flag.
assert hasattr(os, 'closerange') # Added in 2.7
keep = sorted(keep)
min_keep = min(keep)
max_keep = max(keep)
os.closerange(3, min_keep)
os.closerange(max_keep + 1, MAXFD)
for i in xrange(min_keep, max_keep):
if i in keep:
_set_inheritable(i, True)
continue
try:
os.close(i)
except:
pass
def _execute_child(self, args, executable, preexec_fn, close_fds,
pass_fds, cwd, env, universal_newlines,
startupinfo, creationflags, shell,
......@@ -1235,6 +1274,13 @@ class Popen(object):
raise
if self.pid == 0:
# Child
# XXX: Technically we're doing a lot of stuff here that
# may not be safe to do before a exec(), depending on the OS.
# CPython 3 goes to great lengths to precompute a lot
# of this info before the fork and pass it all to C functions that
# try hard not to call things like malloc(). (Of course,
# CPython 2 pretty much did what we're doing.)
try:
# Close parent's pipe ends
if p2cwrite != -1:
......@@ -1254,16 +1300,16 @@ class Popen(object):
errwrite = os.dup(errwrite)
# Dup fds for child
def _dup2(a, b):
def _dup2(existing, desired):
# dup2() removes the CLOEXEC flag but
# we must do it ourselves if dup2()
# would be a no-op (issue #10806).
if a == b:
self._set_cloexec_flag(a, False)
elif a != -1:
os.dup2(a, b)
if existing == desired:
self._set_cloexec_flag(existing, False)
elif existing != -1:
os.dup2(existing, desired)
try:
self._remove_nonblock_flag(b)
self._remove_nonblock_flag(desired)
except OSError:
# Ignore EBADF, it may not actually be
# open yet.
......@@ -1296,21 +1342,7 @@ class Popen(object):
if close_fds:
fds_to_keep = set(pass_fds)
fds_to_keep.add(errpipe_write)
self._close_fds(fds_to_keep)
elif hasattr(os, 'get_inheritable'):
# close_fds was false, and we're on
# Python 3.4 or newer, so "all file
# descriptors except standard streams
# are closed, and inheritable handles
# are only inherited if the close_fds
# parameter is False."
for i in xrange(3, MAXFD):
try:
if i == errpipe_write or os.get_inheritable(i):
continue
os.close(i)
except:
pass
self._close_fds(fds_to_keep, errpipe_write)
if restore_signals:
# restore the documented signals back to sig_dfl;
......
......@@ -119,3 +119,10 @@ BaseTestCase = unittest.TestCase
from greentest.flaky import reraiseFlakyTestTimeout
from greentest.flaky import reraiseFlakyTestRaceCondition
try:
from unittest import mock
except ImportError: # Python 2
import mock
mock = mock
import sys
import os
import errno
import unittest
import time
import gc
import tempfile
import greentest
import gevent
from greentest import mock
from gevent import subprocess
if not hasattr(subprocess, 'mswindows'):
# PyPy3, native python subprocess
subprocess.mswindows = False
import time
import gc
import tempfile
PYPY = hasattr(sys, 'pypy_version_info')
......@@ -236,32 +240,31 @@ class Test(greentest.TestCase):
r = p.stdout.readline()
self.assertEqual(r, b'foobar\n')
if sys.platform != 'win32':
@greentest.ignores_leakcheck
@greentest.skipOnWindows("Not sure why?")
def test_subprocess_in_native_thread(self):
# gevent.subprocess doesn't work from a background
# native thread. See #688
from gevent import monkey
def test_subprocess_in_native_thread(self):
# gevent.subprocess doesn't work from a background
# native thread. See #688
from gevent import monkey
# must be a native thread; defend against monkey-patching
ex = []
Thread = monkey.get_original('threading', 'Thread')
# must be a native thread; defend against monkey-patching
ex = []
Thread = monkey.get_original('threading', 'Thread')
def fn():
with self.assertRaises(TypeError) as exc:
gevent.subprocess.Popen('echo 123', shell=True)
raise AssertionError("Should not be able to construct Popen")
ex.append(exc.exception)
def fn():
with self.assertRaises(TypeError) as exc:
gevent.subprocess.Popen('echo 123', shell=True)
raise AssertionError("Should not be able to construct Popen")
ex.append(exc.exception)
thread = Thread(target=fn)
thread.start()
thread.join()
thread = Thread(target=fn)
thread.start()
thread.join()
self.assertEqual(len(ex), 1)
self.assertTrue(isinstance(ex[0], TypeError), ex)
self.assertEqual(ex[0].args[0], 'child watchers are only available on the default loop')
self.assertEqual(len(ex), 1)
self.assertTrue(isinstance(ex[0], TypeError), ex)
self.assertEqual(ex[0].args[0], 'child watchers are only available on the default loop')
test_subprocess_in_native_thread.ignore_leakcheck = True
@greentest.skipOnLibuvOnPyPyOnWin("hangs")
def __test_no_output(self, kwargs, kind):
......@@ -295,6 +298,91 @@ class Test(greentest.TestCase):
# https://github.com/gevent/gevent/pull/939
self.__test_no_output({}, bytes)
@greentest.skipOnWindows("Testing POSIX fd closing")
class TestFDs(unittest.TestCase):
@mock.patch('os.closerange')
@mock.patch('gevent.subprocess._set_inheritable')
@mock.patch('os.close')
def test_close_fds_brute_force(self, close, set_inheritable, closerange):
keep = (
4, 5,
# Leave a hole
# 6,
7,
)
subprocess.Popen._close_fds_brute_force(keep, None)
closerange.assert_has_calls([
mock.call(3, 4),
mock.call(8, subprocess.MAXFD),
])
set_inheritable.assert_has_calls([
mock.call(4, True),
mock.call(5, True),
])
close.assert_called_once_with(6)
@mock.patch('gevent.subprocess.Popen._close_fds_brute_force')
@mock.patch('os.listdir')
def test_close_fds_from_path_bad_values(self, listdir, brute_force):
listdir.return_value = 'Not an Integer'
subprocess.Popen._close_fds_from_path('path', [], 42)
brute_force.assert_called_once_with([], 42)
@mock.patch('os.listdir')
@mock.patch('os.closerange')
@mock.patch('gevent.subprocess._set_inheritable')
@mock.patch('os.close')
def test_close_fds_from_path(self, close, set_inheritable, closerange, listdir):
keep = (
4, 5,
# Leave a hole
# 6,
7,
)
listdir.return_value = ['1', '6', '37']
subprocess.Popen._close_fds_from_path('path', keep, 5)
self.assertEqual([], closerange.mock_calls)
set_inheritable.assert_has_calls([
mock.call(4, True),
mock.call(7, True),
])
close.assert_has_calls([
mock.call(6),
mock.call(37),
])
@mock.patch('gevent.subprocess.Popen._close_fds_brute_force')
@mock.patch('os.path.isdir')
def test_close_fds_no_dir(self, isdir, brute_force):
isdir.return_value = False
subprocess.Popen._close_fds([], 42)
brute_force.assert_called_once_with([], 42)
isdir.assert_has_calls([
mock.call('/proc/self/fd'),
mock.call('/dev/fd'),
])
@mock.patch('gevent.subprocess.Popen._close_fds_from_path')
@mock.patch('gevent.subprocess.Popen._close_fds_brute_force')
@mock.patch('os.path.isdir')
def test_close_fds_with_dir(self, isdir, brute_force, from_path):
isdir.return_value = True
subprocess.Popen._close_fds([7], 42)
self.assertEqual([], brute_force.mock_calls)
from_path.assert_called_once_with('/proc/self/fd', [7], 42)
class RunFuncTestCase(greentest.TestCase):
# Based on code from python 3.6
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment