Merge.

.hgtags

@@ -78,6 +78,8 @@ a4f75773c0060cee38b0bb651a7aba6f56b0e996 v3.1.3
 32fcb9e94985cb19ce37ba9543f091c0dbe9d7dd v3.1.4rc1
 c918ec9f3a76d6afedfbb5d455004de880443a3d v3.1.4
 ee26aca3219cf4bb0b93352e83edcc9cb28c7802 v3.1.5rc1
+75db2bc69fc9a3e4801e94e3e19801cb096208d8 v3.1.5rc2
+7395330e495ec3316862ca1f6ce0aaf7bdf6785b v3.1.5
 b37b7834757492d009b99cf0ca4d42d2153d7fac v3.2a1
 56d4373cecb73c8b45126ba7b045b3c7b3f94b0b v3.2a2
 da012d9a2c23d144e399d2e01a55b8a83ad94573 v3.2a3

Doc/library/socket.rst

@@ -680,6 +680,16 @@ The module :mod:`socket` exports the following constants and functions:
    .. versionadded:: 3.3
 
 
+.. function:: fromshare(data)
+
+   Instantiate a socket from data obtained from :meth:`~socket.share`.
+   The socket is assumed to be in blocking mode.
+
+   Availability: Windows.
+
+   .. versionadded:: 3.3
+
+
 .. data:: SocketType
 
    This is a Python type object that represents the socket object type. It is the
@@ -1082,6 +1092,21 @@ correspond to Unix system calls applicable to sockets.
    are disallowed.  If *how* is :const:`SHUT_RDWR`, further sends and receives are
    disallowed.
 
+
+.. method:: socket.share(process_id)
+
+    :platform: Windows
+
+    Duplacet a socket and prepare it for sharing with a target process.  The
+    target process must be provided with *process_id*.  The resulting bytes object
+    can then be passed to the target process using some form of interprocess
+    communication and the socket can be recreated there using :func:`fromshare`.
+    Once this method has been called, it is safe to close the socket since
+    the operating system has already duplicated it for the target process.
+
+   .. versionadded:: 3.3
+
+
 Note that there are no methods :meth:`read` or :meth:`write`; use
 :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead.
 

Lib/importlib/test/import_/test_packages.py

@@ -3,6 +3,7 @@ from . import util as import_util
 import sys
 import unittest
 import importlib
+from test import support
 
 
 class ParentModuleTests(unittest.TestCase):
@@ -38,7 +39,10 @@ class ParentModuleTests(unittest.TestCase):
                                          module_code={'mod': module_injection})
         with mock_modules as mock:
             with util.import_state(meta_path=[mock]):
+                try:
                     submodule = import_util.import_(subname)
+                finally:
+                    support.unload(subname)
 
 
 def test_main():

Lib/socket.py

@@ -12,6 +12,7 @@ Functions:
 socket() -- create a new socket object
 socketpair() -- create a pair of new socket objects [*]
 fromfd() -- create a socket object from an open file descriptor [*]
+fromshare() -- create a socket object from data received from socket.share() [*]
 gethostname() -- return the current hostname
 gethostbyname() -- map a hostname to its IP number
 gethostbyaddr() -- map an IP number or hostname to DNS info
@@ -209,7 +210,6 @@ class socket(_socket.socket):
         self._closed = True
         return super().detach()
 
-
 def fromfd(fd, family, type, proto=0):
     """ fromfd(fd, family, type[, proto]) -> socket object
 
@@ -219,6 +219,14 @@ def fromfd(fd, family, type, proto=0):
     nfd = dup(fd)
     return socket(family, type, proto, nfd)
 
+if hasattr(_socket.socket, "share"):
+    def fromshare(info):
+        """ fromshare(info) -> socket object
+
+        Create a socket object from a the bytes object returned by
+        socket.share(pid).
+        """
+        return socket(0, 0, 0, info)
 
 if hasattr(_socket, "socketpair"):
 

Lib/test/test_asyncore.py

@@ -74,15 +74,16 @@ def capture_server(evt, buf, serv):
         pass
     else:
         n = 200
-        while n > 0:
-            r, w, e = select.select([conn], [], [])
+        start = time.time()
+        while n > 0 and time.time() - start < 3.0:
+            r, w, e = select.select([conn], [], [], 0.1)
             if r:
+                n -= 1
                 data = conn.recv(10)
                 # keep everything except for the newline terminator
                 buf.write(data.replace(b'\n', b''))
                 if b'\n' in data:
                     break
-            n -= 1
             time.sleep(0.01)
 
         conn.close()

Lib/test/test_site.py

@@ -39,6 +39,7 @@ class HelperFunctionsTests(unittest.TestCase):
         self.old_base = site.USER_BASE
         self.old_site = site.USER_SITE
         self.old_prefixes = site.PREFIXES
+        self.original_vars = sysconfig._CONFIG_VARS
         self.old_vars = copy(sysconfig._CONFIG_VARS)
 
     def tearDown(self):
@@ -47,7 +48,9 @@ class HelperFunctionsTests(unittest.TestCase):
         site.USER_BASE = self.old_base
         site.USER_SITE = self.old_site
         site.PREFIXES = self.old_prefixes
-        sysconfig._CONFIG_VARS = self.old_vars
+        sysconfig._CONFIG_VARS = self.original_vars
+        sysconfig._CONFIG_VARS.clear()
+        sysconfig._CONFIG_VARS.update(self.old_vars)
 
     def test_makepath(self):
         # Test makepath() have an absolute path for its first return value

Lib/test/test_socket.py

@@ -26,6 +26,10 @@ try:
     import fcntl
 except ImportError:
     fcntl = False
+try:
+    import multiprocessing
+except ImportError:
+    multiprocessing = False
 
 HOST = support.HOST
 MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return
@@ -4643,6 +4647,106 @@ class NonblockConstantTest(unittest.TestCase):
         socket.setdefaulttimeout(t)
 
 
+@unittest.skipUnless(os.name == "nt", "Windows specific")
+@unittest.skipUnless(multiprocessing, "need multiprocessing")
+class TestSocketSharing(SocketTCPTest):
+    # This must be classmethod and not staticmethod or multiprocessing
+    # won't be able to bootstrap it.
+    @classmethod
+    def remoteProcessServer(cls, q):
+        # Recreate socket from shared data
+        sdata = q.get()
+        message = q.get()
+
+        s = socket.fromshare(sdata)
+        s2, c = s.accept()
+
+        # Send the message
+        s2.sendall(message)
+        s2.close()
+        s.close()
+
+    def testShare(self):
+        # Transfer the listening server socket to another process
+        # and service it from there.
+
+        # Create process:
+        q = multiprocessing.Queue()
+        p = multiprocessing.Process(target=self.remoteProcessServer, args=(q,))
+        p.start()
+
+        # Get the shared socket data
+        data = self.serv.share(p.pid)
+
+        # Pass the shared socket to the other process
+        addr = self.serv.getsockname()
+        self.serv.close()
+        q.put(data)
+
+        # The data that the server will send us
+        message = b"slapmahfro"
+        q.put(message)
+
+        # Connect
+        s = socket.create_connection(addr)
+        #  listen for the data
+        m = []
+        while True:
+            data = s.recv(100)
+            if not data:
+                break
+            m.append(data)
+        s.close()
+        received = b"".join(m)
+        self.assertEqual(received, message)
+        p.join()
+
+    def testShareLength(self):
+        data = self.serv.share(os.getpid())
+        self.assertRaises(ValueError, socket.fromshare, data[:-1])
+        self.assertRaises(ValueError, socket.fromshare, data+b"foo")
+
+    def compareSockets(self, org, other):
+        # socket sharing is expected to work only for blocking socket
+        # since the internal python timout value isn't transfered.
+        self.assertEqual(org.gettimeout(), None)
+        self.assertEqual(org.gettimeout(), other.gettimeout())
+
+        self.assertEqual(org.family, other.family)
+        self.assertEqual(org.type, other.type)
+        # If the user specified "0" for proto, then
+        # internally windows will have picked the correct value.
+        # Python introspection on the socket however will still return
+        # 0.  For the shared socket, the python value is recreated
+        # from the actual value, so it may not compare correctly.
+        if org.proto != 0:
+            self.assertEqual(org.proto, other.proto)
+
+    def testShareLocal(self):
+        data = self.serv.share(os.getpid())
+        s = socket.fromshare(data)
+        try:
+            self.compareSockets(self.serv, s)
+        finally:
+            s.close()
+
+    def testTypes(self):
+        families = [socket.AF_INET, socket.AF_INET6]
+        types = [socket.SOCK_STREAM, socket.SOCK_DGRAM]
+        for f in families:
+            for t in types:
+                source = socket.socket(f, t)
+                try:
+                    data = source.share(os.getpid())
+                    shared = socket.fromshare(data)
+                    try:
+                        self.compareSockets(source, shared)
+                    finally:
+                        shared.close()
+                finally:
+                    source.close()
+
+
 def test_main():
     tests = [GeneralModuleTests, BasicTCPTest, TCPCloserTest, TCPTimeoutTest,
              TestExceptions, BufferIOTest, BasicTCPTest2, BasicUDPTest, UDPTimeoutTest ]
@@ -4699,6 +4803,7 @@ def test_main():
         # These are slow when setitimer() is not available
         InterruptedRecvTimeoutTest,
         InterruptedSendTimeoutTest,
+        TestSocketSharing,
     ])
 
     thread_info = support.threading_setup()

Lib/test/test_tools.py

@@ -6,8 +6,10 @@ Tools directory of a Python checkout or tarball, such as reindent.py.
 
 import os
 import sys
+import imp
 import unittest
 import sysconfig
+import tempfile
 from test import support
 from test.script_helper import assert_python_ok
 
@@ -72,6 +74,31 @@ class TestSundryScripts(unittest.TestCase):
                 import analyze_dxp
 
 
+class PdepsTests(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(self):
+        path = os.path.join(scriptsdir, 'pdeps.py')
+        self.pdeps = imp.load_source('pdeps', path)
+
+    @classmethod
+    def tearDownClass(self):
+        if 'pdeps' in sys.modules:
+            del sys.modules['pdeps']
+
+    def test_process_errors(self):
+        # Issue #14492: m_import.match(line) can be None.
+        with tempfile.TemporaryDirectory() as tmpdir:
+            fn = os.path.join(tmpdir, 'foo')
+            with open(fn, 'w') as stream:
+                stream.write("#!/this/will/fail")
+            self.pdeps.process(fn, {})
+
+    def test_inverse_attribute_error(self):
+        # Issue #14492: this used to fail with an AttributeError.
+        self.pdeps.inverse({'a': []})
+
+
 def test_main():
     support.run_unittest(*[obj for obj in globals().values()
                                if isinstance(obj, type)])

Misc/NEWS

@@ -19,6 +19,8 @@ Core and Builtins
 Library
 -------
 
+- Don't Py_DECREF NULL variable in io.IncrementalNewlineDecoder.
+
 - Issue #8515: Set __file__ when run file in IDLE.
   Initial patch by Bruce Frederiksen.
 
@@ -230,6 +232,9 @@ Library
 - Issue #14210: pdb now has tab-completion not only for command names, but
   also for their arguments, wherever possible.
 
+- Issue #14310: Sockets can now be with other processes on Windows using
+  the api socket.socket.share() and socket.fromshare().
+
 Build
 -----
 

Modules/_io/textio.c

@@ -460,7 +460,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self,
             output = PyUnicode_FromKindAndData(kind, translated, out);
             PyMem_Free(translated);
             if (!output)
-                goto error;
+                return NULL;
         }
         self->seennl |= seennl;
     }

Modules/socketmodule.c

@@ -3771,6 +3771,34 @@ PyDoc_STRVAR(sock_ioctl_doc,
 Control the socket with WSAIoctl syscall. Currently supported 'cmd' values are\n\
 SIO_RCVALL:  'option' must be one of the socket.RCVALL_* constants.\n\
 SIO_KEEPALIVE_VALS:  'option' is a tuple of (onoff, timeout, interval).");
+#endif
+
+#if defined(MS_WINDOWS)
+static PyObject*
+sock_share(PySocketSockObject *s, PyObject *arg)
+{
+    WSAPROTOCOL_INFO info;
+    DWORD processId;
+    int result;
+
+    if (!PyArg_ParseTuple(arg, "I", &processId))
+        return NULL;
+
+    Py_BEGIN_ALLOW_THREADS
+    result = WSADuplicateSocket(s->sock_fd, processId, &info);
+    Py_END_ALLOW_THREADS
+    if (result == SOCKET_ERROR)
+        return set_error();
+    return PyBytes_FromStringAndSize((const char*)&info, sizeof(info));
+}
+PyDoc_STRVAR(sock_share_doc,
+"share(process_id) -> bytes\n\
+\n\
+Share the socket with another process.  The target process id\n\
+must be provided and the resulting bytes object passed to the target\n\
+process.  There the shared socket can be instantiated by calling\n\
+socket.fromshare().");
+
 
 #endif
 
@@ -3802,6 +3830,10 @@ static PyMethodDef sock_methods[] = {
 #if defined(MS_WINDOWS) && defined(SIO_RCVALL)
     {"ioctl",             (PyCFunction)sock_ioctl, METH_VARARGS,
                       sock_ioctl_doc},
+#endif
+#if defined(MS_WINDOWS)
+    {"share",         (PyCFunction)sock_share, METH_VARARGS,
+                      sock_share_doc},
 #endif
     {"listen",            (PyCFunction)sock_listen, METH_O,
                       listen_doc},
@@ -3930,6 +3962,32 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds)
         return -1;
 
     if (fdobj != NULL && fdobj != Py_None) {
+#ifdef MS_WINDOWS
+        /* recreate a socket that was duplicated */
+        if (PyBytes_Check(fdobj)) {
+            WSAPROTOCOL_INFO info;
+            if (PyBytes_GET_SIZE(fdobj) != sizeof(info)) {
+                PyErr_Format(PyExc_ValueError,
+                    "socket descriptor string has wrong size, "
+                    "should be %zu bytes.", sizeof(info));
+                return -1;
+            }
+            memcpy(&info, PyBytes_AS_STRING(fdobj), sizeof(info));
+            Py_BEGIN_ALLOW_THREADS
+            fd = WSASocket(FROM_PROTOCOL_INFO, FROM_PROTOCOL_INFO,
+                     FROM_PROTOCOL_INFO, &info, 0, WSA_FLAG_OVERLAPPED);
+            Py_END_ALLOW_THREADS
+            if (fd == INVALID_SOCKET) {
+                set_error();
+                return -1;
+            }
+            family = info.iAddressFamily;
+            type = info.iSocketType;
+            proto = info.iProtocol;
+        }
+        else
+#endif
+        {
             fd = PyLong_AsSocket_t(fdobj);
             if (fd == (SOCKET_T)(-1) && PyErr_Occurred())
                 return -1;
@@ -3939,6 +3997,7 @@ sock_initobj(PyObject *self, PyObject *args, PyObject *kwds)
                 return -1;
             }
         }
+    }
     else {
         Py_BEGIN_ALLOW_THREADS
         fd = socket(family, type, proto);

Tools/scripts/pdeps.py

@@ -76,10 +76,9 @@ def process(filename, table):
             nextline = fp.readline()
             if not nextline: break
             line = line[:-1] + nextline
-        if m_import.match(line) >= 0:
-            (a, b), (a1, b1) = m_import.regs[:2]
-        elif m_from.match(line) >= 0:
-            (a, b), (a1, b1) = m_from.regs[:2]
+        m_found = m_import.match(line) or m_from.match(line)
+        if m_found:
+            (a, b), (a1, b1) = m_found.regs[:2]
         else: continue
         words = line[a1:b1].split(',')
         # print '#', line, words
@@ -87,6 +86,7 @@ def process(filename, table):
             word = word.strip()
             if word not in list:
                 list.append(word)
+    fp.close()
 
 
 # Compute closure (this is in fact totally general)
@@ -123,7 +123,7 @@ def closure(table):
 def inverse(table):
     inv = {}
     for key in table.keys():
-        if not inv.has_key(key):
+        if key not in inv:
             inv[key] = []
         for item in table[key]:
             store(inv, item, key)