Python 2, subprocess: Let unbuffered binary writes to popen.stdin loop to write all the data.

More like what Python 2 standard library does. Beware of relying on that though during an upgrade.

Fixes #1711

docs/changes/1711.bugfix

+Python 2: Make ``gevent.subprocess.Popen.stdin`` objects have a
+``write`` method that guarantees to write the entire argument in
+binary, unbuffered mode. This may require multiple trips around the
+event loop, but more closely matches the behaviour of the Python 2
+standard library (and gevent prior to 1.5). The number of bytes
+written is still returned (instead of ``None``).

src/gevent/_fileobjectposix.py

@@ -14,6 +14,7 @@ from gevent._compat import reraise
 from gevent._fileobjectcommon import cancel_wait_ex
 from gevent._fileobjectcommon import FileObjectBase
 from gevent._fileobjectcommon import OpenDescriptor
+from gevent._fileobjectcommon import WriteIsWriteallMixin
 from gevent._hub_primitives import wait_on_watcher
 from gevent.hub import get_hub
 from gevent.os import _read
@@ -213,27 +214,40 @@ class GreenFileDescriptorIO(RawIOBase):
         )
 
 
+class GreenFileDescriptorIOWriteall(WriteIsWriteallMixin,
+                                    GreenFileDescriptorIO):
+    pass
+
+
 class GreenOpenDescriptor(OpenDescriptor):
 
-    def open_raw(self):
+    def _do_open_raw(self):
         if self.is_fd():
-            fileio = GreenFileDescriptorIO(self.fobj, self, closefd=self.closefd)
+            fileio = GreenFileDescriptorIO(self._fobj, self, closefd=self.closefd)
         else:
-            closefd = False
             # Either an existing file object or a path string (which
             # we open to get a file object). In either case, the other object
             # owns the descriptor and we must not close it.
             closefd = False
-            if hasattr(self.fobj, 'fileno'):
-                raw = self.fobj
-            else:
-                raw = OpenDescriptor.open_raw(self)
+
+            raw = OpenDescriptor._do_open_raw(self)
 
             fileno = raw.fileno()
             fileio = GreenFileDescriptorIO(fileno, self, closefd=closefd)
             fileio._keep_alive = raw
         return fileio
 
+    def _make_atomic_write(self, result, raw):
+        # Our return value from _do_open_raw is always a new
+        # object that we own, so we're always free to change
+        # the class.
+        assert result is not raw or self._raw_object_is_new(raw)
+        if result.__class__ is GreenFileDescriptorIO:
+            result.__class__ = GreenFileDescriptorIOWriteall
+        else:
+            result = OpenDescriptor._make_atomic_write(self, result, raw)
+        return result
+
 
 class FileObjectPosix(FileObjectBase):
     """
@@ -309,9 +323,9 @@ class FileObjectPosix(FileObjectBase):
 
     def __init__(self, *args, **kwargs):
         descriptor = GreenOpenDescriptor(*args, **kwargs)
+        FileObjectBase.__init__(self, descriptor)
         # This attribute is documented as available for non-blocking reads.
-        self.fileio, buffered_fobj = descriptor.open_raw_and_wrapped()
-        FileObjectBase.__init__(self, buffered_fobj, descriptor.closefd)
+        self.fileio = descriptor.opened_raw()
 
     def _do_close(self, fobj, closefd):
         try:

src/gevent/subprocess.py

@@ -455,6 +455,14 @@ def FileObject(*args, **kwargs):
     # Defer importing FileObject until we need it
     # to allow it to be configured more easily.
     from gevent.fileobject import FileObject as _FileObject
+    if not PY3:
+        # Make write behave like the old Python 2 file
+        # write and loop to consume output, even when not
+        # buffered.
+        __FileObject = _FileObject
+        def _FileObject(*args, **kwargs):
+            kwargs['atomic_write'] = True
+            return __FileObject(*args, **kwargs)
     globals()['FileObject'] = _FileObject
     return _FileObject(*args)
 
@@ -557,6 +565,12 @@ class Popen(object):
     .. seealso:: :class:`subprocess.Popen`
        This class should have the same interface as the standard library class.
 
+    .. caution::
+
+       The default values of some arguments, notably ``buffering``, differ
+       between Python 2 and Python 3. For the most consistent behaviour across
+       versions, it's best to explicitly pass the desired values.
+
     .. caution::
 
        On Python 2, the ``read`` method of the ``stdout`` and ``stderr`` attributes
@@ -602,6 +616,15 @@ class Popen(object):
        Add the *group*, *extra_groups*, *user*, and *umask* arguments. These
        were added to Python 3.9, but are available in any gevent version, provided
        the underlying platform support is present.
+
+    .. versionchanged:: NEXT
+       On Python 2 only, if unbuffered binary communication is requested,
+       the ``stdin`` attribute of this object will have a ``write`` method that
+       actually performs internal buffering and looping, similar to the standard library.
+       It guarantees to write all the data given to it in a single call (but internally
+       it may make many system calls and/or trips around the event loop to accomplish this).
+       See :issue:`1711`.
+
     """
 
     if GenericAlias is not None:
@@ -751,6 +774,7 @@ class Popen(object):
                                         encoding=self.encoding, errors=self.errors)
             else:
                 self.stdin = FileObject(p2cwrite, 'wb', bufsize)
+
         if c2pread != -1:
             if universal_newlines or text_mode:
                 if PY3:

src/gevent/tests/test__fileobject.py

@@ -11,6 +11,10 @@ import unittest
 import gevent
 from gevent import fileobject
 from gevent._fileobjectcommon import OpenDescriptor
+try:
+    from gevent._fileobjectposix import GreenOpenDescriptor
+except ImportError:
+    GreenOpenDescriptor = None
 
 from gevent._compat import PY2
 from gevent._compat import PY3
@@ -387,8 +391,11 @@ class TestTextMode(unittest.TestCase):
 
 class TestOpenDescriptor(greentest.TestCase):
 
+    def _getTargetClass(self):
+        return OpenDescriptor
+
     def _makeOne(self, *args, **kwargs):
-        return OpenDescriptor(*args, **kwargs)
+        return self._getTargetClass()(*args, **kwargs)
 
     def _check(self, regex, kind, *args, **kwargs):
         with self.assertRaisesRegex(kind, regex):
@@ -411,14 +418,33 @@ class TestOpenDescriptor(greentest.TestCase):
         vase('take a newline', mode='rb', newline='\n'),
     )
 
+    def test_atomicwrite_fd(self):
+        from gevent._fileobjectcommon import WriteallMixin
+        # It basically only does something when buffering is otherwise disabled
+        desc = self._makeOne(1, 'wb',
+                             buffering=0,
+                             closefd=False,
+                             atomic_write=True)
+        self.assertTrue(desc.atomic_write)
+
+        fobj = desc.opened()
+        self.assertIsInstance(fobj, WriteallMixin)
+
 def pop():
     for regex, kind, kwargs in TestOpenDescriptor.CASES:
         setattr(
-            TestOpenDescriptor, 'test_' + regex,
+            TestOpenDescriptor, 'test_' + regex.replace(' ', '_'),
             lambda self, _re=regex, _kind=kind, _kw=kwargs: self._check(_re, _kind, 1, **_kw)
         )
 pop()
 
+@unittest.skipIf(GreenOpenDescriptor is None, "No support for non-blocking IO")
+class TestGreenOpenDescripton(TestOpenDescriptor):
+    def _getTargetClass(self):
+        return GreenOpenDescriptor
+
+
+
 
 if __name__ == '__main__':
     greentest.main()